static void Main(string[] args) { try { CommandLineArguments cmd = new CommandLineArguments(); if(CommandLine.Parser.ParseArgumentsWithUsage(args, cmd)) { Random random = new Random(cmd.seed); FindStepLib fs = new FindStepLib(cmd.convex, random, cmd.verbose); DCGScorer.truncLevel = cmd.truncLevel; if (cmd.selfTest) { //fs.alphaPos = false; // Set to false to search for the optimal negative alpha. Default is true. SelfTest(10, 100, fs); } else { QueryCollection qc1 = new QueryCollection(cmd.firstScoresFile, cmd.labelForUnlabeled, cmd.skipDegenerateQueries, cmd.scoreForDegenerateQuery); QueryCollection qc2 = new QueryCollection(cmd.secondScoresFile, cmd.labelForUnlabeled, cmd.skipDegenerateQueries, cmd.scoreForDegenerateQuery); // Assume that the first 'feature' is in fact the scores qc1.AssignScoresFromFeature(0); qc2.AssignScoresFromFeature(0); double bestGain; fs.FindStep(qc1, qc2, null, out bestGain); } #if DEBUG // Force console to stick around Console.WriteLine("...Press Enter to terminate program..."); Console.ReadLine(); #endif } } catch(Exception exc) { Console.WriteLine(exc.Message); } }
/// <summary> /// /// </summary> /// <param name="qc1">the scores for all the queries computed from the existing system</param> /// <param name="qc2">the score for all the queries computed from the newly added tree</param> /// <param name="queryIdxActive">the set of "active queries" that we use to find the optimal combination</param> /// <param name="bestMeanNDCGGain"></param> /// <returns></returns> public double FindStep(QueryCollection qc1, QueryCollection qc2, int[] queryIdxActive, out double bestMeanNDCGGain) { PairRankedItems.Reset(); // (queryIdxActive == null) <=> using all the queries in qc1/qc2 int cActiveQueries = (queryIdxActive == null) ? qc1.NQueries : queryIdxActive.Length; if(qc1.Count != qc2.Count) throw new Exception("Input files must have same number of rows."); if(qc1.NQueries != qc2.NQueries) throw new Exception("Input files must have same number of queries."); if (qc1.NQueries < cActiveQueries) throw new Exception("Active queries must be less than all the queries."); DCGScorer dcg = new DCGScorer(); // The relabeling must be done before FillRankedItems is called // REVIEW: ?? long nPairs; // Only used for debug int nDocs; // ditto CountDocsNAllPairs(qc1, qc2, queryIdxActive, out nPairs, out nDocs); // ditto int rankedItemCtr = 0; //PairRankedItems pri = null; int nQueries = 0; int nSkippedQueries = 0; for (int i = 0; i < cActiveQueries; ++i) { int qIdx = (queryIdxActive == null) ? i : queryIdxActive[i]; Query query1 = qc1.queries[qIdx]; Query query2 = qc2.queries[qIdx]; // We discard the array itself each time, but the object pointers persist. // Also: discard any queries that have maxDCG = 0. RankedItem[] thisRankedItems = FillRankedItems(query1, query2, dcg, random); if(thisRankedItems != null) { FillRanks(thisRankedItems); //pri = FillPairRankedItems(thisRankedItems, convex, maxStep, ref rankedItemCtr); FillPairRankedItems(thisRankedItems, convex, alphaPos, maxStep, ref rankedItemCtr); // This forms a linked list. ++nQueries; } else { ++nSkippedQueries; } } PairRankedItems[] pairRankedItems = PRI_ListToArray(); if (alphaPos) { Array.Sort(pairRankedItems, new SortPairRankedItemsIncreasing()); // First value closest to zero, next more positive } else { Array.Sort(pairRankedItems, new SortPairRankedItemsDecreasing()); // First value still closest to zero, next more negative } // Now that we have the sorted values of alpha: compute which global alpha gives best NDCG gain. double bestAlpha; FindBestAlpha(pairRankedItems, dcg, nQueries, out bestAlpha, out bestMeanNDCGGain); if (verbose) { Console.WriteLine("{0} queries total, {1} skipped queries, {2} docs", nQueries + nSkippedQueries, nSkippedQueries, nDocs); Console.WriteLine("Tot. # pairs = {0}, num. pairs in computation = {1}", nPairs, pairRankedItems.Length); // For the convex combination, it's tempting to rescale alpha so that the first weight is one. But this is not always possible: // it may need to be -1. Console.WriteLine("Best mean NDCG Gain = {0}, best alpha = {1}", bestMeanNDCGGain, bestAlpha); // Check that the gain is correct. qc1.ComputeNDCGs(); qc2.ComputeNDCGs(); double firstFactor = convex ? 1.0 - bestAlpha : 1.0; QueryCollection qc = QueryCollection.LinearlyCombine(firstFactor, qc1, bestAlpha, qc2); qc.ComputeNDCGs(); Console.WriteLine("NON-TRUNC: First NDCG = {0:F4}/{1:F4}, second = {2:F4}/{3:F4}, combined = {4:F4}/{5:F4}", qc1.NonTruncNDCG_pes, qc1.NonTruncNDCG_opt, qc2.NonTruncNDCG_pes, qc2.NonTruncNDCG_opt, qc.NonTruncNDCG_pes, qc.NonTruncNDCG_opt); Console.WriteLine(" TRUNC: First NDCG = {0:F4}/{1:F4}, second = {2:F4}/{3:F4}, combined = {4:F4}/{5:F4}", qc1.TruncNDCG_pes, qc1.TruncNDCG_opt, qc2.TruncNDCG_pes, qc2.TruncNDCG_opt, qc.TruncNDCG_pes, qc.TruncNDCG_opt); } return bestAlpha; }
/// <summary> /// The only reason we pass both data1 and data2 is to change the labels in the second one, too. /// </summary> /// <param name="data1"></param> /// <param name="data2"></param> /// <param name="labelForUnlabeled"></param> /// <returns></returns> //static public void Relabel(IRankDataCollection data1, IRankDataCollection data2, float labelForUnlabeled) //{ // IEnumerator ienumData1 = data1.GetEnumerator(); // IEnumerator ienumData2 = data2.GetEnumerator(); // while(ienumData1.MoveNext()) // { // ienumData2.MoveNext(); // RankData query1 = (RankData)ienumData1.Current; // RankData query2 = (RankData)ienumData2.Current; // for(int i = 0; i < query1.Labels.Length; ++i) // { // if(query1.Labels[i] == -1) // query1.Labels[i] = labelForUnlabeled; // if(query2.Labels[i] == -1) // query2.Labels[i] = labelForUnlabeled; // } // } //} /// <summary> /// Debug only: how does the total number of pairs compare with the number of pairs used in the computation? /// </summary> /// <param name="data1"></param> /// <param name="data2"></param> /// <param name="labelForUnlabeled"></param> /// <returns></returns> public void CountDocsNAllPairs(QueryCollection qc1, QueryCollection qc2, int[] queryIdxActive, out long nPairs, out int nDocs) { nPairs = 0; nDocs = 0; int cActiveQuery = (queryIdxActive == null) ? qc1.queries.Length : queryIdxActive.Length; for (int i = 0; i < cActiveQuery; ++i) { int qIdx = (queryIdxActive == null) ? i : queryIdxActive[i]; int tot = qc1.queries[qIdx].Labels.Length; Debug.Assert(tot == qc2.queries[qIdx].Labels.Length, "query collection size mismatch"); nDocs += tot; nPairs += ( tot * ( tot - 1 ) ) / 2; } }
public double BestCombinedMeanNDCG(QueryCollection qc1, QueryCollection qc2) { double bestNDCGGain; double alpha = FindStep(qc1, qc2, null, out bestNDCGGain); QueryCollection qc = QueryCollection.LinearlyCombine(1.0, qc1, alpha, qc2); qc.ComputeNDCGs(); if (verbose) { // Print all three NDCGs: previous model, current model, combination qc1.ComputeNDCGs(); qc2.ComputeNDCGs(); Console.WriteLine("Previous model: NDCG = {0:F6}-{1:F6}-{2:F6}, NDCG@{3} = {4:F6}-{5:F6}-{6:F6}", qc1.NonTruncNDCG_pes, qc1.NonTruncNDCG_mean, qc1.NonTruncNDCG_opt, DCGScorer.truncLevel, qc1.TruncNDCG_pes, qc1.TruncNDCG_mean, qc1.TruncNDCG_opt); Console.WriteLine("Current model: NDCG = {0:F6}-{1:F6}-{2:F6}, NDCG@{3} = {4:F6}-{5:F6}-{6:F6}", qc2.NonTruncNDCG_pes, qc2.NonTruncNDCG_mean, qc2.NonTruncNDCG_opt, DCGScorer.truncLevel, qc2.TruncNDCG_pes, qc2.TruncNDCG_mean, qc2.TruncNDCG_opt); Console.WriteLine("Combined model: NDCG = {0:F6}-{1:F6}-{2:F6}, NDCG@{3} = {4:F6}-{5:F6}-{6:F6}", qc.NonTruncNDCG_pes, qc.NonTruncNDCG_mean, qc.NonTruncNDCG_opt, DCGScorer.truncLevel, qc.TruncNDCG_pes, qc.TruncNDCG_mean, qc.TruncNDCG_opt); Console.WriteLine("alpha = {0}", alpha); } return qc.NonTruncNDCG_mean; }
public void AssignScoresFrom(QueryCollection qc) { if (qc.NQueries != NQueries) throw new Exception("QueryCollection.AssignScores: query array length mismatch"); for (int iQuery = 0; iQuery < NQueries; ++iQuery) { Query qIn = qc.queries[iQuery]; Query q = queries[iQuery]; if (qIn.scores.Length != q.scores.Length) throw new Exception("QueryCollection.AssignScores: query scores length mismatch"); for (int iScore = 0; iScore < q.Length; ++iScore) { q.scores[iScore] = qIn.scores[iScore]; } } }
/// <summary> /// Same, but replace the scores in qc2 with the results. /// </summary> /// <param name="alpha"></param> /// <param name="qc1"></param> /// <param name="beta"></param> /// <param name="qc2"></param> public static void LinearlyCombineNUpdate(double alpha, QueryCollection qc1, double beta, QueryCollection qc2, QueryCollection qcTarget) { #if DEBUG if(qc1.nQueries != qc2.nQueries || qc1.NQueries != qcTarget.NQueries) throw new Exception("LinearlyCombineNUpdate: queries size mismatch"); #endif for (int iQuery = 0; iQuery < qc1.nQueries; ++iQuery) { double[] scores1 = qc1.queries[iQuery].scores; double[] scores2 = qc2.queries[iQuery].scores; double[] scores3 = qcTarget.queries[iQuery].scores; #if DEBUG if (scores1.Length != scores2.Length || scores1.Length != scores3.Length) throw new Exception("LinearlyCombineNUpdate: scores size mismatch"); #endif for (int iScore = 0; iScore < scores1.Length; ++iScore) { scores3[iScore] = alpha * scores1[iScore] + beta * scores2[iScore]; } } }
/// <summary> /// Same, but linearly combine several vectors into one, and increment qcTarget accordingly. /// It is up to the calling code to zero out qcTarget if desired. /// </summary> /// <param name="weights"></param> /// <param name="qcArray"></param> /// <param name="start">Index of first query to use.</param> /// <param name="end">Index of last query to use.</param> /// <param name="qc2"></param> /// <returns></returns> public static void LinearlyCombineNIncrement(double[] weights, QueryCollection[] qcArray, int start, int end, QueryCollection qcTarget) { if(start > end || start < 0 || end >= qcArray.Length) throw new Exception("Illegal indices passed to LinearlyCombine"); if(weights.Length != qcArray.Length) throw new Exception("weights must have same length as qcArray"); for(int j = start; j <= end; ++j) { QueryCollection qcSource = qcArray[j]; double wt = weights[j]; for(int i = 0; i < qcTarget.nQueries; ++i) { double[] scoresSource = qcSource.queries[i].scores; double[] scoresTarget = qcTarget.queries[i].scores; if(scoresSource.Length != scoresTarget.Length) throw new Exception("LinearlyCombine: scores size mismatch"); for(int k = 0; k < scoresSource.Length; ++k) { scoresTarget[k] += wt * scoresSource[k]; } } } }
/// <summary> /// Use one function rather than operators, since it's much more efficient. /// </summary> /// <param name="alpha"></param> /// <param name="qc1"></param> /// <param name="beta"></param> /// <param name="qc2"></param> /// <returns></returns> public static QueryCollection LinearlyCombine(double alpha, QueryCollection qc1, double beta, QueryCollection qc2) { QueryCollection qc = qc1.CopyEmptyQueryCollection(); for (int i = 0; i < qc.nQueries; ++i) { double[] scores1 = qc1.queries[i].scores; double[] scores2 = qc2.queries[i].scores; double[] scoresCombined = qc.queries[i].scores; if (scores1.Length != scores2.Length) throw new Exception("LinearlyCombine: size mismatch"); for (int j = 0; j < scores1.Length; ++j) { scoresCombined[j] = alpha * scores1[j] + beta * scores2[j]; } } return qc; }
/// <summary> /// Generate two QueryCollections containing randomly generated scores and labels (although they share /// all the same labels, as though one dataset tested on two models). The scores are loosely /// correlated with labels. Then, compute the best linear combination. Finally compare the claimed NDCG gain /// with the NDCG gain computed directly. The relative frequencies of the labels are taken from the May 2005 /// training set: /// /// Perfect: 0.0204 /// Excellent: 0.0523 /// Good: 0.2714 /// Fair: 0.2855 /// Bad: 0.3704 /// /// Note we use random features to make it very unlikely that there will be any degeneracy: so the claimed delta NDCG /// should be what's actually measured by taking the linear combination that FindStep proposes. /// </summary> /// <param name="qc1"></param> /// <param name="qc2"></param> /// <param name="nDocsPerQuery"></param> static void SelfTest(int nQueries, int nDocsPerQuery, FindStepLib fs) { Random rangen = new Random(0); float[] priors = new float[5]; priors[0] = 0.3704F; // bads first priors[1] = 0.2855F; priors[2] = 0.2714F; priors[3] = 0.0523F; priors[4] = 0.0204F; double scale1 = 10.0; double scale2 = 20.0; int nScores = 1; QueryCollection qc1 = new QueryCollection(nQueries, priors, scale1, nScores, nDocsPerQuery, rangen); // Must share labels QueryCollection qc2 = qc1.CopyEmptyQueryCollection(); for(int i = 0; i < qc2.queries.Length; ++i) { Query q1 = qc1.queries[i]; Query q2 = qc2.queries[i]; for(int j = 0; j < q1.Length; ++j) { double label = (double) q1.Labels[j]; if (q2.Labels[j] != label) throw new Exception("Labels mismatch."); q1.scores[j] = (float)(label + scale1 * (2.0 * rangen.NextDouble() - 1.0)); q2.scores[j] = (float)(label + scale2 * (2.0 * rangen.NextDouble() - 1.0)); } } double bestMeanNDCGGain; // We will only check for positive alphas. double alpha = fs.FindStep(qc1, qc2, null, out bestMeanNDCGGain); // prints out the best NDCG gain Console.WriteLine("Optimal alpha = {0}", alpha); double firstFactor = fs.convex ? (1.0 - alpha) : 1.0; qc1.ComputeNDCGs(); double initialNDCG_pes = qc1.NonTruncNDCG_pes; double initialNDCG_opt = qc1.NonTruncNDCG_opt; Console.WriteLine("Initial nonTruncNDCG = {0}-{1}", initialNDCG_pes, initialNDCG_opt); QueryCollection qc = QueryCollection.LinearlyCombine(firstFactor, qc1, alpha, qc2); qc.ComputeNDCGs(); double finalNDCG_pes = qc.NonTruncNDCG_pes; double finalNDCG_opt = qc.NonTruncNDCG_opt; Console.WriteLine("Final nonTruncNDCG = {0}-{1}", finalNDCG_pes, finalNDCG_opt); Console.WriteLine("Type RETURN for exhaustive search"); Console.ReadLine(); double bestFound = 0.0; double maxAlpha = fs.convex ? 1.0 : fs.MaxStep; double alphaFactor = fs.alphaPos ? 1.0 : -1.0; for(int i = 0; i < 10001; ++i) { alpha = alphaFactor * (double)(i * maxAlpha) / 10000.0; qc = QueryCollection.LinearlyCombine(firstFactor, qc1, alpha, qc2); qc.ComputeNDCGs(); if (qc.NonTruncNDCG_opt != qc.NonTruncNDCG_pes) throw new Exception("Self test requires no degeneracy"); double finalNDCG_mean = qc.NonTruncNDCG_mean; if(finalNDCG_mean > bestFound) { Console.WriteLine("Best NDCG found so far with search: alpha = {0}, NDCG = {1}", alpha, finalNDCG_mean); bestFound = finalNDCG_mean; } } }
/// <summary> /// Constructor of BoostTreeRankNetLoss /// </summary> /// <param name="dp">the input data used to train boosted regression trees</param> /// <param name="learnRate">the input learning rate specified in training</param> public BoostTreeNoGainLambdaLoss(LabelFeatureDataCoded labelFeatureDataCoded, LabelConverter labelConvert, float learnRate, float labelWeight, StepSizeType stepSizeType, FindStepLib fs, float labelForUnlabeled, double scoreForDegenerateQuery, int truncLevel) { this.learnRate = learnRate; this.labelWeight = labelWeight; this.labelForUnlabeled = labelForUnlabeled; this.scoreForDegenerateQuery = scoreForDegenerateQuery; this.truncLevel = truncLevel; this.labels = new float[labelFeatureDataCoded.NumDataPoint]; for (int i = 0; i < labelFeatureDataCoded.NumDataPoint; i++) { this.labels[i] = labelConvert.convert(labelFeatureDataCoded.GetLabel(i)); } this.numSamples = labels.Length; this.score = new float[this.numSamples]; this.funValue = new float[this.numSamples]; this.pseudoResponses = new float[this.numSamples]; this.weights = new float[this.numSamples]; this.labelFeatureDataCoded = labelFeatureDataCoded; //data member to compute the optimal adjustment factor (step size) for leaf nodes response this.qcAccum = this.CreateQueryCollection(); this.qcCurrent = this.CreateQueryCollection(); this.fs = fs; this.stepSizeType = stepSizeType; }
private QueryCollection CreateQueryCollection() { int cQueris = this.labelFeatureDataCoded.DataGroups.GroupCounts; Query[] queries = new Query[cQueris]; for (int qIdx = 0; qIdx < cQueris; qIdx++) { DataGroup queryGroup = this.labelFeatureDataCoded.DataGroups[qIdx]; Query query = CreateQuery(queryGroup, this.labels, this.score, this.labelForUnlabeled, this.scoreForDegenerateQuery); queries[qIdx] = query; } bool skipDegenerateQueries = true; QueryCollection qc = new QueryCollection(queries, skipDegenerateQueries, this.scoreForDegenerateQuery); return qc; }