public override void AddScores(RegressionTree tree, double multiplier) { _k++; double coeff = (_k - 1.0) / (_k + 2.0); int innerLoopSize = 1 + Dataset.NumDocs / BlockingThreadPool.NumThreads; // +1 is to make sure we don't have a few left over at the end // REVIEW: This partitioning doesn't look optimal. // Probably make sence to investigate better ways of splitting data? var actions = new Action[(int)Math.Ceiling(1.0 * Dataset.NumDocs / innerLoopSize)]; var actionIndex = 0; for (int d = 0; d < Dataset.NumDocs; d += innerLoopSize) { var fromDoc = d; var toDoc = Math.Min(d + innerLoopSize, Dataset.NumDocs); actions[actionIndex++] = () => { var featureBins = Dataset.GetFeatureBinRowwiseIndexer(); for (int doc = fromDoc; doc < toDoc; doc++) { double output = multiplier * tree.GetOutput(featureBins[doc]); double newXK = YK[doc] + output; double newYK = newXK + coeff * (newXK - XK[doc]); XK[doc] = newXK; YK[doc] = newYK; } }; } Parallel.Invoke(new ParallelOptions { MaxDegreeOfParallelism = BlockingThreadPool.NumThreads }, actions); SendScoresUpdatedMessage(); }
public void GetOutputs(Dataset dataset, double[] outputs, int prefix) { if (prefix > _trees.Count || prefix < 0) { prefix = _trees.Count; } int innerLoopSize = 1 + dataset.NumDocs / BlockingThreadPool.NumThreads; // minimize number of times we have to skip forward in the sparse arrays // REVIEW: This partitioning doesn't look optimal. // Probably make sence to investigate better ways of splitting data? var actions = new Action[(int)Math.Ceiling(1.0 * dataset.NumDocs / innerLoopSize)]; var actionIndex = 0; for (int d = 0; d < dataset.NumDocs; d += innerLoopSize) { actions[actionIndex++] = () => { var featureBins = dataset.GetFeatureBinRowwiseIndexer(); var toDoc = Math.Min(d + innerLoopSize, dataset.NumDocs); for (int doc = d; doc < toDoc; doc++) { outputs[doc] = GetOutput(featureBins[doc], prefix); } }; } Parallel.Invoke(new ParallelOptions { MaxDegreeOfParallelism = BlockingThreadPool.NumThreads }, actions); }
/// <summary> /// Constructs partitioning object based on the documents and RegressionTree splits /// NOTE: It has been optimized for speed and multiprocs with 10x gain on naive LINQ implementation /// </summary> public DocumentPartitioning(RegressionTree tree, Dataset dataset) : this(dataset.NumDocs, tree.NumLeaves) { using (Timer.Time(TimerEvent.DocumentPartitioningConstruction)) { // figure out which leaf each document belongs to // NOTE: break it up into NumThreads chunks. This minimizes the number of re-computations necessary in // the row-wise indexer. int innerLoopSize = 1 + dataset.NumDocs / BlockingThreadPool.NumThreads; // +1 is to make sure we don't have a few left over at the end // figure out the exact number of chunks, needed in pathological cases when NumDocs < NumThreads int numChunks = dataset.NumDocs / innerLoopSize; if (dataset.NumDocs % innerLoopSize != 0) { ++numChunks; } var perChunkDocumentLists = new List <int> [numChunks][]; // REVIEW: This partitioning doesn't look optimal. // Probably make sence to investigate better ways of splitting data? var actions = new Action[(int)Math.Ceiling(1.0 * dataset.NumDocs / innerLoopSize)]; var actionIndex = 0; for (int docStart = 0; docStart < dataset.NumDocs; docStart += innerLoopSize) { var fromDoc = docStart; var toDoc = Math.Min(docStart + innerLoopSize, dataset.NumDocs); var chunkIndex = docStart / innerLoopSize; actions[actionIndex++] = () => { Contracts.Assert(perChunkDocumentLists[chunkIndex] == null); var featureBins = dataset.GetFeatureBinRowwiseIndexer(); List <int>[] perLeafDocumentLists = Enumerable.Range(0, tree.NumLeaves) .Select(x => new List <int>(innerLoopSize / tree.NumLeaves)) .ToArray(); for (int d = fromDoc; d < toDoc; d++) { int leaf = tree.GetLeaf(featureBins[d]); perLeafDocumentLists[leaf].Add(d); } perChunkDocumentLists[chunkIndex] = perLeafDocumentLists; }; } Parallel.Invoke(new ParallelOptions { MaxDegreeOfParallelism = BlockingThreadPool.NumThreads }, actions); // establish leaf starts and document counts _leafCount = Enumerable.Range(0, tree.NumLeaves) .Select(leaf => Enumerable.Range(0, perChunkDocumentLists.Length) .Select(thread => perChunkDocumentLists[thread][leaf].Count) .Sum()) .ToArray(); var cumulativeLength = _leafCount.CumulativeSum <int>().Take(tree.NumLeaves - 1); _leafBegin = Enumerable.Range(0, 1).Concat(cumulativeLength).ToArray(); // move all documents that belong to the same leaf together Contracts.Assert(_documents.Length == _leafBegin[tree.NumLeaves - 1] + _leafCount[tree.NumLeaves - 1]); actions = new Action[tree.NumLeaves]; actionIndex = 0; for (int leaf = 0; leaf < tree.NumLeaves; leaf++) { var l = leaf; actions[actionIndex++] = () => { int documentPos = _leafBegin[l]; for (int chunkIndex = 0; chunkIndex < perChunkDocumentLists.Length; chunkIndex++) { foreach (int d in perChunkDocumentLists[chunkIndex][l]) { _documents[documentPos++] = d; } perChunkDocumentLists[chunkIndex][l] = null; } }; } Parallel.Invoke(new ParallelOptions { MaxDegreeOfParallelism = BlockingThreadPool.NumThreads }, actions); } }