/// <summary>
/// Use one function rather than operators, since it's much more efficient.
/// </summary>
/// <param name="alpha"></param>
/// <param name="qc1"></param>
/// <param name="beta"></param>
/// <param name="qc2"></param>
/// <returns></returns>
public static QueryCollection LinearlyCombine(double alpha, QueryCollection qc1, double beta, QueryCollection qc2)
{
QueryCollection qc = qc1.CopyEmptyQueryCollection();
for (int i = 0; i < qc.nQueries; ++i)
{
double[] scores1 = qc1.queries[i].scores;
double[] scores2 = qc2.queries[i].scores;
double[] scoresCombined = qc.queries[i].scores;
if (scores1.Length != scores2.Length)
throw new Exception("LinearlyCombine: size mismatch");
for (int j = 0; j < scores1.Length; ++j)
{
scoresCombined[j] = alpha * scores1[j] + beta * scores2[j];
}
}
return qc;
}
/// <summary>
/// Generate two QueryCollections containing randomly generated scores and labels (although they share
/// all the same labels, as though one dataset tested on two models). The scores are loosely
/// correlated with labels. Then, compute the best linear combination. Finally compare the claimed NDCG gain
/// with the NDCG gain computed directly. The relative frequencies of the labels are taken from the May 2005
/// training set:
///
/// Perfect: 0.0204
/// Excellent: 0.0523
/// Good: 0.2714
/// Fair: 0.2855
/// Bad: 0.3704
///
/// Note we use random features to make it very unlikely that there will be any degeneracy: so the claimed delta NDCG
/// should be what's actually measured by taking the linear combination that FindStep proposes.
/// </summary>
/// <param name="qc1"></param>
/// <param name="qc2"></param>
/// <param name="nDocsPerQuery"></param>
static void SelfTest(int nQueries, int nDocsPerQuery, FindStepLib fs)
{
Random rangen = new Random(0);
float[] priors = new float[5];
priors[0] = 0.3704F; // bads first
priors[1] = 0.2855F;
priors[2] = 0.2714F;
priors[3] = 0.0523F;
priors[4] = 0.0204F;
double scale1 = 10.0;
double scale2 = 20.0;
int nScores = 1;
QueryCollection qc1 = new QueryCollection(nQueries, priors, scale1, nScores, nDocsPerQuery, rangen);
// Must share labels
QueryCollection qc2 = qc1.CopyEmptyQueryCollection();
for(int i = 0; i < qc2.queries.Length; ++i)
{
Query q1 = qc1.queries[i];
Query q2 = qc2.queries[i];
for(int j = 0; j < q1.Length; ++j)
{
double label = (double) q1.Labels[j];
if (q2.Labels[j] != label)
throw new Exception("Labels mismatch.");
q1.scores[j] = (float)(label + scale1 * (2.0 * rangen.NextDouble() - 1.0));
q2.scores[j] = (float)(label + scale2 * (2.0 * rangen.NextDouble() - 1.0));
}
}
double bestMeanNDCGGain;
// We will only check for positive alphas.
double alpha = fs.FindStep(qc1, qc2, null, out bestMeanNDCGGain); // prints out the best NDCG gain
Console.WriteLine("Optimal alpha = {0}", alpha);
double firstFactor = fs.convex ? (1.0 - alpha) : 1.0;
qc1.ComputeNDCGs();
double initialNDCG_pes = qc1.NonTruncNDCG_pes;
double initialNDCG_opt = qc1.NonTruncNDCG_opt;
Console.WriteLine("Initial nonTruncNDCG = {0}-{1}", initialNDCG_pes, initialNDCG_opt);
QueryCollection qc = QueryCollection.LinearlyCombine(firstFactor, qc1, alpha, qc2);
qc.ComputeNDCGs();
double finalNDCG_pes = qc.NonTruncNDCG_pes;
double finalNDCG_opt = qc.NonTruncNDCG_opt;
Console.WriteLine("Final nonTruncNDCG = {0}-{1}", finalNDCG_pes, finalNDCG_opt);
Console.WriteLine("Type RETURN for exhaustive search");
Console.ReadLine();
double bestFound = 0.0;
double maxAlpha = fs.convex ? 1.0 : fs.MaxStep;
double alphaFactor = fs.alphaPos ? 1.0 : -1.0;
for(int i = 0; i < 10001; ++i)
{
alpha = alphaFactor * (double)(i * maxAlpha) / 10000.0;
qc = QueryCollection.LinearlyCombine(firstFactor, qc1, alpha, qc2);
qc.ComputeNDCGs();
if (qc.NonTruncNDCG_opt != qc.NonTruncNDCG_pes)
throw new Exception("Self test requires no degeneracy");
double finalNDCG_mean = qc.NonTruncNDCG_mean;
if(finalNDCG_mean > bestFound)
{
Console.WriteLine("Best NDCG found so far with search: alpha = {0}, NDCG = {1}", alpha, finalNDCG_mean);
bestFound = finalNDCG_mean;
}
}
}