public void Train(ILabeledExampleCollection <LblT, ExT> dataset) { Preconditions.CheckNotNull(dataset); var trainDataset = new LabeledDataset <LblT, ExT>(dataset); for (int i = 0; i < mInnerModels.Length; i++) { if (mInnerModels[i] == null) { mInnerModels[i] = CreateModel(i); } mInnerModels[i].Train(GetTrainSet(i, mInnerModels[i], trainDataset)); } foreach (LabeledExample <LblT, ExT> le in trainDataset) { LabeledExample <LblT, ExT> le_ = le; string key = StringOf(mInnerModels.Select(m => m.Predict(le_.Example).BestClassLabel)); VotingEntry votingEntry = mVotingEntries[key]; votingEntry.LabelCounts[le.Label]++; } foreach (VotingEntry entry in mVotingEntries.Values) { PerformVoting(entry); } IsTrained = true; }
public void Train(ILabeledExampleCollection <LblT, ExT> dataset) { Preconditions.CheckNotNull(dataset); var trainDataset = new LabeledDataset <LblT, ExT>(dataset); for (int i = 0; i < mInnerModels.Length; i++) { if (mInnerModels[i] == null) { mInnerModels[i] = CreateModel(i); } mInnerModels[i].Train(GetTrainSet(i, mInnerModels[i], trainDataset)); } foreach (LabeledExample <LblT, ExT> le in trainDataset) { LabeledExample <LblT, ExT> le_ = le; double[] scores = GetPredictionScores(mInnerModels.Select(m => m.Predict(le_.Example)).ToArray()).ToArray(); mTagDistrTable.AddCount(le.Label, scores); } mTagDistrTable.Calculate(); IsTrained = true; }
public static LabeledDataset <SentimentLabel, SparseVector <double> > InitBowSpace(BowSpace bowSpace, IEnumerable <LabeledExample <SentimentLabel, string> > labeledExamples, IEnumerable <string> initExamples = null) { LabeledExample <SentimentLabel, string>[] examples = labeledExamples as LabeledExample <SentimentLabel, string>[] ?? labeledExamples.ToArray(); List <SparseVector <double> > bowData; if (initExamples != null) { Preconditions.CheckArgument(!(bowSpace is DeltaBowSpace <SentimentLabel>)); bowSpace.Initialize(initExamples); bowData = examples.Select(le => bowSpace.ProcessDocument(le.Example)).ToList(); } else { bowData = bowSpace is DeltaBowSpace <SentimentLabel> ?((DeltaBowSpace <SentimentLabel>)bowSpace).Initialize(new LabeledDataset <SentimentLabel, string>(examples)) : bowSpace.Initialize(examples.Select(d => d.Example)); } var bowDataset = new LabeledDataset <SentimentLabel, SparseVector <double> >(); for (int i = 0; i < bowData.Count; i++) { bowDataset.Add(examples[i].Label, bowData[i]); } return(bowDataset); }
public static void SaveTab(string[] featureNames, LabeledDataset <BlogMetaData, SparseVector <double> > dataset, ClassType classType, string fileName) { using (StreamWriter w = new StreamWriter(fileName, /*append=*/ false, Encoding.ASCII)) { for (int i = 0; i < featureNames.Length; i++) { w.Write(featureNames[i] + "\t"); } w.WriteLine("author"); for (int i = 0; i < featureNames.Length; i++) { w.Write("c\t"); } w.WriteLine("d"); for (int i = 0; i < featureNames.Length; i++) { w.Write("\t"); } w.WriteLine("class"); foreach (LabeledExample <BlogMetaData, SparseVector <double> > lblEx in dataset) { foreach (string lblStr in AnalysisUtils.GetLabel(lblEx.Label, classType).Split(',')) { if (lblStr != "") { foreach (IdxDat <double> item in lblEx.Example) { w.Write(item.Dat + "\t"); } w.WriteLine(lblStr); } } } } }
public static void SaveArff(string[] featureNames, LabeledDataset <BlogMetaData, SparseVector <double> > dataset, ClassType classType, string fileName) { using (StreamWriter w = new StreamWriter(fileName, /*append=*/ false, Encoding.ASCII)) { w.WriteLine("@RELATION r" + Guid.NewGuid().ToString("N")); w.WriteLine(); foreach (string featureName in featureNames) { w.WriteLine("@ATTRIBUTE " + featureName + " NUMERIC"); } w.Write("@ATTRIBUTE class "); ArrayList <string> classes = new ArrayList <string>(); ((IEnumerable <LabeledExample <BlogMetaData, SparseVector <double> > >)dataset).ToList().ForEach( x => classes.AddRange(AnalysisUtils.GetLabel(x.Label, classType).Split(','))); classes = new ArrayList <string>(classes.Distinct()); w.WriteLine(classes.ToString().Replace("( ", "{").Replace(" )", "}").Replace(" ", ",")); w.WriteLine(); w.WriteLine("@DATA"); foreach (LabeledExample <BlogMetaData, SparseVector <double> > lblEx in dataset) { foreach (string lblStr in AnalysisUtils.GetLabel(lblEx.Label, classType).Split(',')) { if (lblStr != "") { foreach (IdxDat <double> item in lblEx.Example) { w.Write(item.Dat + ","); } w.WriteLine(lblStr); } } } } }
public void Train(ILabeledExampleCollection <LblT, ExT> dataset) { var binaryDataset = new LabeledDataset <LblT, ExT>(dataset.Select(le => new LabeledExample <LblT, ExT>(le.Label.Equals(OneLabel) ? OneLabel : OtherLabel, le.Example))); mBinaryModel.Train(binaryDataset); IsTrained = true; }
public void TestSortedCheckFail() { LabeledDataset <int, int> testSet, trainSet; LabeledDataset <int, int> ld = NewData(new[, ] { { 1, 10 }, { 2, 1 }, { 1, 1 } }); ld.SplitForStratifiedCrossValidation(2, 1, out trainSet, out testSet); }
private Model TrainModel(LabeledDataset <SentimentLabel, SparseVector <double> > dataset, SentimentLabel label, SentimentLabel otherLabel1, SentimentLabel otherLabel2) { IModel <SentimentLabel, SparseVector <double> > model = CreateModel(); var otherLabelWeight1 = (double)dataset.Count(le => le.Label == otherLabel1) / dataset.Count(le => le.Label != label); var otherLabelWeight2 = (double)dataset.Count(le => le.Label == otherLabel2) / dataset.Count(le => le.Label != label); dataset = new LabeledDataset <SentimentLabel, SparseVector <double> >(dataset.Select(le => new LabeledExample <SentimentLabel, SparseVector <double> >(le.Label == label ? label : otherLabel1, le.Example))); var scores = new List <double>(); var scoresOthers = new List <double>(); var validation = new CrossValidator <SentimentLabel, SparseVector <double> > { NumFolds = NumTrainFolds, Dataset = dataset, OnAfterPrediction = (sender, foldN, m, ex, le, prediction) => { if (le.Label == prediction.BestClassLabel) { if (prediction.BestClassLabel == label) { scores.Add(prediction.BestScore); } else { scoresOthers.Add(prediction.BestScore); } } return(true); } }; validation.Models.Add(model); validation.Run(); // train model model.Train(dataset); return(new Model { InnerModel = model, Weight = validation.PerfData.GetSumPerfMatrix(validation.ExpName, validation.GetModelName(model)).GetMacroF1(), Label = label, OtherLabel1 = otherLabel1, OtherLabelWeight1 = otherLabelWeight1, OtherLabel2 = otherLabel2, OtherLabelWeight2 = otherLabelWeight2, Scores = scores.OrderBy(s => s).ToArray(), ScoresOthers = scoresOthers.OrderBy(s => s).ToArray() }); }
public void Train(ILabeledExampleCollection <LblT, ExT> dataset) { foreach (ModelLabel modelLabel in ModelLabels.Take(ModelLabels.Count() - 1)) { modelLabel.Model.Train(dataset); ModelLabel modelLabel_ = modelLabel; dataset = new LabeledDataset <LblT, ExT>(dataset.Where(le => !le.Label.Equals(modelLabel_.Label))); } ModelLabels.Last().Model.Train(dataset); IsTrained = true; }
static void GetExtremes <T>(LabeledDataset <T, SparseVector <double> > dataset, out SparseVector <double> minValues, out SparseVector <double> maxValues) { minValues = new SparseVector <double>(); maxValues = new SparseVector <double>(); int maxIdx = ((IEnumerableList <SparseVector <double> >)dataset).Max(x => x.Max(y => y.Idx)); for (int featureIdx = 0; featureIdx <= maxIdx; featureIdx++) { minValues[featureIdx] = ((IEnumerableList <SparseVector <double> >)dataset).Min(x => x[featureIdx]); maxValues[featureIdx] = ((IEnumerableList <SparseVector <double> >)dataset).Max(x => x[featureIdx]); } }
public override void Train(ILabeledExampleCollection <SentimentLabel, SparseVector <double> > dataset) { Preconditions.CheckNotNull(dataset); var ds = new LabeledDataset <SentimentLabel, SparseVector <double> >(dataset .Select(le => new LabeledExample <SentimentLabel, SparseVector <double> >(le.Label, le.Example))); mPosModel = TrainModel(ds, SentimentLabel.Positive, SentimentLabel.Negative, SentimentLabel.Neutral); mNegModel = TrainModel(ds, SentimentLabel.Negative, SentimentLabel.Positive, SentimentLabel.Neutral); mNeuModel = TrainModel(ds, SentimentLabel.Neutral, SentimentLabel.Positive, SentimentLabel.Negative); IsTrained = true; }
static LabeledDataset <string, SparseVector <double> > CreateSingleFeatureDataset(LabeledDataset <BlogMetaData, SparseVector <double> > srcDataset, ClassType classType, int fIdx) { SparseVector <double> minValues, maxValues; GetExtremes(srcDataset, out minValues, out maxValues); LabeledDataset <string, SparseVector <double> > dataset = new LabeledDataset <string, SparseVector <double> >(); ((IEnumerable <LabeledExample <BlogMetaData, SparseVector <double> > >)srcDataset).ToList() .ForEach(x => dataset.Add(new LabeledExample <string, SparseVector <double> >(AnalysisUtils.GetLabel(x.Label, classType), new SparseVector <double>( new double[] { (x.Example[fIdx] - minValues[fIdx]) / (maxValues[fIdx] - minValues[fIdx]) } // simple normalization )))); return(dataset); }
public override sealed void Train(ILabeledExampleCollection <SentimentLabel, SparseVector <double> > dataset) { Preconditions.CheckNotNull(dataset); var labeledDataset = (LabeledDataset <SentimentLabel, SparseVector <double> >)dataset; var trainDataset = new LabeledDataset <SentimentLabel, SparseVector <double> >(labeledDataset.Where(le => le.Label != SentimentLabel.Neutral)); if (mBinaryClassifier == null) { mBinaryClassifier = CreateModel(); mBinaryClassifier.Train(trainDataset); } IsTrained = true; /*Calculate positive and negative average distances*/ int positiveTweetsNumber = 0; int negativeTweetsNumber = 0; PosAverageDistance = 0; NegAverageDistance = 0; foreach (LabeledExample <SentimentLabel, SparseVector <double> > example in trainDataset) { Prediction <SentimentLabel> prediction = mBinaryClassifier.Predict(example.Example); SentimentLabel bestLabelPredicted = prediction.BestClassLabel; double bestScorePredicted = bestLabelPredicted == SentimentLabel.Negative ? -prediction.BestScore : prediction.BestScore; SentimentLabel actualLabel = example.Label; if (actualLabel == SentimentLabel.Positive) { PosAverageDistance += bestScorePredicted; positiveTweetsNumber++; } else if (actualLabel == SentimentLabel.Negative) { NegAverageDistance += bestScorePredicted; negativeTweetsNumber++; } } PosAverageDistance = PosAverageDistance / positiveTweetsNumber; NegAverageDistance = NegAverageDistance / negativeTweetsNumber; }
private static LabeledDataset <int, int> NewData(int[,] labelCounts, bool sortShuffled = false) { var result = new LabeledDataset <int, int>(); for (int i = 0, k = 1; i <= labelCounts.GetUpperBound(0); i++) { int label = labelCounts[i, 0], count = labelCounts[i, 1]; for (int j = 0; j < count; j++) { result.Add(label, k++); } } if (sortShuffled) { result.GroupLabels(true); } return(result); }
public override void Train(ILabeledExampleCollection <SentimentLabel, SparseVector <double> > dataset) { Preconditions.CheckNotNull(dataset); var posDataset = new LabeledDataset <SentimentLabel, SparseVector <double> >(dataset.Select(le => new LabeledExample <SentimentLabel, SparseVector <double> >(le.Label == SentimentLabel.Positive ? SentimentLabel.Positive : SentimentLabel.Negative, le.Example))); mPosClassifier = CreateModel(); mPosClassifier.Train(posDataset); var negDataset = new LabeledDataset <SentimentLabel, SparseVector <double> >(dataset.Select(le => new LabeledExample <SentimentLabel, SparseVector <double> >(le.Label == SentimentLabel.Negative ? SentimentLabel.Negative : SentimentLabel.Positive, le.Example))); mNegClassifier = CreateModel(); mNegClassifier.Train(negDataset); if (PosBiasCalibration != null || NegBiasCalibration != null) { var labeledDataset = new LabeledDataset <SentimentLabel, SparseVector <double> >(dataset); double?posBias = Calibrate(true, labeledDataset); double?negBias = Calibrate(false, labeledDataset); BiasToPosRate = posBias ?? BiasToPosRate; BiasToNegRate = negBias ?? BiasToNegRate; } mPosSortedScores = mNegSortedScores = null; mExampleScores = dataset.Select(le => { Prediction <SentimentLabel> posPrediction = mPosClassifier.Predict(le.Example); Prediction <SentimentLabel> negPrediction = mNegClassifier.Predict(le.Example); return(new ExampleScore { Label = le.Label, PosScore = posPrediction.BestClassLabel == SentimentLabel.Positive ? posPrediction.BestScore : -posPrediction.BestScore, NegScore = negPrediction.BestClassLabel == SentimentLabel.Negative ? -negPrediction.BestScore : negPrediction.BestScore }); }).ToArray(); UpdateDistrTable(); IsTrained = true; }
private Model TrainModel(LabeledDataset <SentimentLabel, SparseVector <double> > dataset, SentimentLabel label1, SentimentLabel label2) { IModel <SentimentLabel, SparseVector <double> > model = CreateModel(); var scores1 = new List <double>(); var scores2 = new List <double>(); var validation = new CrossValidator <SentimentLabel, SparseVector <double> > { NumFolds = NumTrainFolds, Dataset = dataset, OnAfterPrediction = (sender, foldN, m, ex, le, prediction) => { if (le.Label == prediction.BestClassLabel) { if (prediction.BestClassLabel == label1) { scores1.Add(prediction.BestScore); } else if (prediction.BestClassLabel == label2) { scores2.Add(prediction.BestScore); } } return(true); } }; validation.Models.Add(model); validation.Run(); // train model model.Train(dataset); return(new Model { InnerModel = model, Label1 = label1, Label2 = label2, Scores1 = scores1.OrderBy(s => s).ToArray(), Scores2 = scores2.OrderBy(s => s).ToArray(), Weight = validation.PerfData.GetSumPerfMatrix(validation.ExpName, validation.GetModelName(model)).GetMacroF1() }); }
public ValidationTask(TaskContext taskContext) { Context = Preconditions.CheckNotNull(taskContext); LabeledExample <SentimentLabel, string>[] labeledExamples = taskContext.DataSource.GetData().ToArray(); TaskUtils.ProcessFeatures(taskContext, labeledExamples); var labeledDataset = new LabeledDataset <SentimentLabel, string>(labeledExamples); // lazy model creation IEnumerable <Func <IModel <SentimentLabel, SparseVector <double> > > > modelFacotry = Enumerable.Range(0, taskContext.Models.Length) .Select <int, Func <IModel <SentimentLabel, SparseVector <double> > > >(i => () => taskContext.ModelFactory(i)); Validator = new FoldLocalBowCrossValidator <SentimentLabel>(modelFacotry) { Dataset = labeledDataset, BowSpaceFunc = taskContext.BowSpaceFactory, ModelNameFunc = (sender, m) => taskContext.GetModelName(m) }; }
private double?Calibrate(bool doPosPlane, LabeledDataset <SentimentLabel, SparseVector <double> > dataset) { BiasCalibration calibration = doPosPlane ? PosBiasCalibration : NegBiasCalibration; if (calibration == null) { return(null); } Preconditions.CheckArgument(calibration.BiasStep > 0); Preconditions.CheckNotNull(calibration.OptimizationFunc); double maxScore = double.MinValue; double optimalBias = 0; var biasScorePairs = calibration.IsSaveBiasScorePairs ? new List <Tuple <double, double> >() : null; for (double bias = calibration.BiasLowerBound; bias <= calibration.BiasUpperBound; bias += calibration.BiasStep) { var matrix = new PerfMatrix <SentimentLabel>(null); foreach (LabeledExample <SentimentLabel, SparseVector <double> > le in dataset) { Prediction <SentimentLabel> prediction = PredictInternal(le.Example, doPosPlane ? bias : 0, doPosPlane ? 0 : bias); matrix.AddCount(le.Label, prediction.BestClassLabel); } double score = calibration.OptimizationFunc(matrix); if (score > maxScore) { maxScore = score; optimalBias = bias; } if (biasScorePairs != null) { biasScorePairs.Add(new Tuple <double, double>(bias, score)); } Console.WriteLine("{0}\t{1:0.000}\t{2:0.000}", doPosPlane, bias, score); } if (biasScorePairs != null) { calibration.BiasScorePairs = biasScorePairs.ToArray(); } return(optimalBias); }
public void TestEvenlyDistributed() { int size = DatasetSize; for (int numLabels = 2; numLabels <= size / 2; numLabels++) { var labelCounts = new int[numLabels, 2]; for (int label = 1; label <= numLabels; label++) { int segSize = size / numLabels; if (label <= size % numLabels) { segSize++; } labelCounts[label - 1, 0] = label; labelCounts[label - 1, 1] = segSize; } double labelDistr = 1.0 / numLabels; LabeledDataset <int, int> ld = NewData(labelCounts, true); for (int numFolds = 2; numFolds <= size / numLabels; numFolds++) { var aggTestSet = new LabeledDataset <int, int>(); for (int i = 0; i < numFolds; i++) { LabeledDataset <int, int> trainSet, testSet; ld.SplitForStratifiedCrossValidation(numFolds, i + 1, out trainSet, out testSet); AssertSetEquality(trainSet.Concat(testSet), ld); aggTestSet.AddRange(testSet); foreach (double distr in testSet.GroupBy(le => le.Label).Select(g => (double)g.Count() / testSet.Count)) { Assert.IsTrue(Math.Abs(labelDistr - distr) <= 1.0 / testSet.Count); } foreach (double distr in trainSet.GroupBy(le => le.Label).Select(g => (double)g.Count() / trainSet.Count)) { Assert.IsTrue(Math.Abs(labelDistr - distr) <= 1.0 / trainSet.Count); } } AssertSetEquality(aggTestSet, ld); } } }
protected override ILabeledDataset <LblT, SparseVector <double> > MapTrainSet(int foldN, ILabeledDataset <LblT, string> trainSet) { BowSpace bowSpace; Preconditions.CheckState(!mFoldBowSpaces.TryGetValue(foldN, out bowSpace)); Preconditions.CheckState(mFoldBowSpaces.TryAdd(foldN, bowSpace = BowSpaceFunc())); List <SparseVector <double> > bowData = bowSpace is DeltaBowSpace <LblT> ?((DeltaBowSpace <LblT>)bowSpace).Initialize(trainSet) : bowSpace.Initialize(trainSet.Select(d => d.Example)); var bowDataset = new LabeledDataset <LblT, SparseVector <double> >(); for (int i = 0; i < bowData.Count; i++) { bowDataset.Add(trainSet[i].Label, bowData[i]); } return(bowDataset); }
public override void Train(ILabeledExampleCollection <SentimentLabel, SparseVector <double> > dataset) { Preconditions.CheckState(BowSpace != null); var replDataset = new LabeledDataset <SentimentLabel, SparseVector <double> >(); foreach (LabeledExample <SentimentLabel, SparseVector <double> > le in dataset) { SparseVector <double> vector1, vector2; Replicate(le.Example, out vector1, out vector2); replDataset.Add(new LabeledExample <SentimentLabel, SparseVector <double> >( le.Label == SentimentLabel.Neutral ? SentimentLabel.Negative : le.Label, vector1)); replDataset.Add(new LabeledExample <SentimentLabel, SparseVector <double> >( le.Label == SentimentLabel.Neutral ? SentimentLabel.Positive : le.Label, vector2)); } mClassifier = CreateModel(); mClassifier.Train(replDataset); IsTrained = true; }
public void TestGroupedCheckOk() { LabeledDataset <int, int> testSet, trainSet; LabeledDataset <int, int> ld = NewData(new[, ] { { 1, 10 }, { 2, 1 } }); ld.SplitForStratifiedCrossValidation(2, 1, out trainSet, out testSet); ld = NewData(new[, ] { { 1, 10 }, { 2, 1 }, { 1, 1 } }); ld.GroupLabels(true); ld.SplitForStratifiedCrossValidation(2, 1, out trainSet, out testSet); ld = NewData(new[, ] { { 1, 10 }, { 2, 1 }, { 1, 1 }, { 2, 10 } }); ld.GroupLabels(true); ld.SplitForStratifiedCrossValidation(2, 1, out trainSet, out testSet); }
public void TrainModels(IEnumerable <Author> authors) { foreach (Author author in authors) { LabeledDataset <string, SparseVector <double> > ds = new LabeledDataset <string, SparseVector <double> >(); foreach (Author otherAuthor in authors) { if (otherAuthor != author && !otherAuthor.mIsTagged) { foreach (Text text in otherAuthor.mTexts) { ds.Add(new LabeledExample <string, SparseVector <double> >(otherAuthor.mName, text.mFeatureVectors[mSelector])); } } } SvmMulticlassClassifier <string> model = new SvmMulticlassClassifier <string>(); model.C = Convert.ToDouble(Utils.GetConfigValue("SvmMultiClassC", "5000")); model.Train(ds); mModels.Add(author.mName, model); } }
public void TestFolding() { for (int size = 2; size <= DatasetSize; size++) { LabeledDataset <int, int> ld = NewData(new[, ] { { 1, size } }, true); for (int numFolds = 2; numFolds <= size; numFolds++) { var aggTestSet = new LabeledDataset <int, int>(); for (int i = 0; i < numFolds; i++) { LabeledDataset <int, int> trainSet, testSet; ld.SplitForStratifiedCrossValidation(numFolds, i + 1, out trainSet, out testSet); AssertSetEquality(trainSet.Concat(testSet), ld); aggTestSet.AddRange(testSet); } AssertSetEquality(aggTestSet, ld); } } }
public void Train(ILabeledExampleCollection <LblT, string> dataset) { Preconditions.CheckState(!IsTrained); Preconditions.CheckNotNull(dataset); Preconditions.CheckNotNull(BowSpace); Preconditions.CheckNotNull(FeatureProcessor); Preconditions.CheckNotNull(Model); // preprocess the text foreach (LabeledExample <LblT, string> le in dataset) { le.Example = FeatureProcessor.Run(le.Example); } // bow vectors List <SparseVector <double> > bowData = BowSpace is DeltaBowSpace <LblT> ?(BowSpace as DeltaBowSpace <LblT>).Initialize(dataset as ILabeledDataset <LblT, string> ?? new LabeledDataset <LblT, string>(dataset)) : BowSpace.Initialize(dataset.Select(d => d.Example)); var bowDataset = new LabeledDataset <LblT, SparseVector <double> >(); for (int i = 0; i < bowData.Count; i++) { bowDataset.Add(dataset[i].Label, bowData[i]); } // train if (OnTrainModel == null) { Model.Train(bowDataset); } else { OnTrainModel(this, bowDataset); } IsTrained = true; }
public override void Run(object[] args) { // prepare data IStemmer stemmer; Set <string> .ReadOnly stopWords; TextMiningUtils.GetLanguageTools(Language.English, out stopWords, out stemmer); // Create a tokenizer. var tokenizer = new UnicodeTokenizer { MinTokenLen = 2, // Each token must be at least 2 characters long. Filter = TokenizerFilter.AlphaStrict // Tokens can consist of alphabetic characters only. }; // take data for two classes from cvs file var data = new List <LabeledTweet>(GetLabeledTweets().Where(lt => lt.Polarity != 2)).ToList(); // Create a bag-of-words space. var bowSpc = new BowSpace { Tokenizer = tokenizer, // Assign the tokenizer. StopWords = stopWords, // Assign the stop words. Stemmer = stemmer, // Assign the stemmer. MinWordFreq = 1, // A term must appear at least n-times in the corpus for it to be part of the vocabulary. MaxNGramLen = 2, // Terms consisting of at most n-consecutive words will be considered. WordWeightType = WordWeightType.TermFreq, // Set the weighting scheme for the bag-of-words vectors to TF. //WordWeightType = WordWeightType.TfIdf, // Set the weighting scheme for the bag-of-words vectors to TF-IDF. NormalizeVectors = true, // The TF-IDF vectors will be normalized. CutLowWeightsPerc = 0 // The terms with the lowest weights, summing up to 20% of the overall weight sum, will be removed from each TF-IDF vector. }; ArrayList <SparseVector <double> > bowData = bowSpc.Initialize(data.Select(d => d.Text)); // label data var labeledSet = new LabeledDataset <string, SparseVector <double> >(); for (int i = 0; i < data.Count; i++) { labeledSet.Add(data[i].Label, bowData[i]); } labeledSet.Shuffle(); int testSize = labeledSet.Count / 10; var trainingSet = new LabeledDataset <string, SparseVector <double> >(labeledSet.Skip(testSize)); var testSet = new LabeledDataset <string, SparseVector <double> >(labeledSet.Take(testSize)); //-------------------- SVM var svmBinClass = new SvmBinaryClassifier <string> { VerbosityLevel = SvmLightVerbosityLevel.Off }; if (args.Any()) { svmBinClass.C = (int)args[0]; } //svmBinClass.BiasedHyperplane = true; //svmBinClass.CustomParams = "-t 3"; // non-linear kernel //svmBinClass.CustomParams = String.Format("-j {0}",j); svmBinClass.Train(trainingSet); int correct = 0; double avgDist = 0; foreach (LabeledExample <string, SparseVector <double> > labeledExample in testSet) { var prediction = svmBinClass.Predict(labeledExample.Example); //Output.WriteLine("actual: {0}\tpredicted: {1}\t score: {2:0.0000}", labeledExample.Label, prediction.BestClassLabel, prediction.BestScore); avgDist += prediction.BestScore; if (prediction.BestClassLabel == labeledExample.Label) { correct++; } } Output.WriteLine("Accuracy: {0:0.00}", 100.0 * correct / testSet.Count); Output.WriteLine("Avg. distance: {0:0.00}", avgDist / testSet.Count); Result.Add("accuracy", (double)correct / testSet.Count); Result.Add("classifier", svmBinClass); Result.Add("labeled_data", labeledSet); }
protected abstract LabeledDataset <LblT, ExT> GetTrainSet(int modelIdx, IModel <LblT, ExT> model, LabeledDataset <LblT, ExT> trainSet);
public Vector2D[] ComputeLayout(LayoutSettings settings) { // clustering mLogger.Info("ComputeLayout", "Clustering ..."); mKMeans = new IncrementalKMeans(mKClust); mKMeans.Eps = mKMeansEps; mKMeans.Random = mRandom; mKMeans.Trials = 3; ClusteringResult clustering = mKMeans.Cluster(mDataset); // throws ArgumentValueException // determine reference instances UnlabeledDataset <SparseVector <double> > dsRefInst = new UnlabeledDataset <SparseVector <double> >(); foreach (SparseVector <double> centroid in mKMeans.GetCentroids()) { dsRefInst.Add(centroid); // dataset of reference instances mDataset.Add(centroid); // add centroids to the main dataset } // position reference instances mLogger.Info("ComputeLayout", "Positioning reference instances ..."); SparseMatrix <double> simMtx = ModelUtils.GetDotProductSimilarity(dsRefInst, mSimThresh, /*fullMatrix=*/ false); StressMajorizationLayout sm = new StressMajorizationLayout(dsRefInst.Count, new DistFunc(simMtx)); sm.Random = mRandom; sm.MaxSteps = int.MaxValue; sm.MinDiff = 0.00001; mRefPos = sm.ComputeLayout(); // k-NN mLogger.Info("ComputeLayout", "Computing similarities ..."); simMtx = ModelUtils.GetDotProductSimilarity(mDataset, mSimThresh, /*fullMatrix=*/ true); mLogger.Info("ComputeLayout", "Constructing system of linear equations ..."); LabeledDataset <double, SparseVector <double> > lsqrDs = new LabeledDataset <double, SparseVector <double> >(); mPatches = new ArrayList <Patch>(mDataset.Count); for (int i = 0; i < mDataset.Count; i++) { mPatches.Add(new Patch(i)); } foreach (IdxDat <SparseVector <double> > simMtxRow in simMtx) { if (simMtxRow.Dat.Count <= 1) { mLogger.Warn("ComputeLayout", "Instance #{0} has no neighborhood.", simMtxRow.Idx); } ArrayList <KeyDat <double, int> > knn = new ArrayList <KeyDat <double, int> >(simMtxRow.Dat.Count); foreach (IdxDat <double> item in simMtxRow.Dat) { if (item.Idx != simMtxRow.Idx) { knn.Add(new KeyDat <double, int>(item.Dat, item.Idx)); } } knn.Sort(DescSort <KeyDat <double, int> > .Instance); int count = Math.Min(knn.Count, mKNnExt); for (int i = 0; i < count; i++) { mPatches[simMtxRow.Idx].List.Add(new KeyDat <double, Patch>(knn[i].Key, mPatches[knn[i].Dat])); } mPatches[simMtxRow.Idx].ProcessList(); count = Math.Min(knn.Count, mKNn); SparseVector <double> eq = new SparseVector <double>(); double wgt = 1.0 / (double)count; for (int i = 0; i < count; i++) { eq.InnerIdx.Add(knn[i].Dat); eq.InnerDat.Add(-wgt); } eq.InnerIdx.Sort(); // *** sort only indices eq[simMtxRow.Idx] = 1; lsqrDs.Add(0, eq); } Vector2D[] layout = new Vector2D[mDataset.Count - mKClust]; for (int i = mDataset.Count - mKClust, j = 0; i < mDataset.Count; i++, j++) { SparseVector <double> eq = new SparseVector <double>(new IdxDat <double>[] { new IdxDat <double>(i, 1) }); lsqrDs.Add(mRefPos[j].X, eq); } LSqrModel lsqr = new LSqrModel(); lsqr.Train(lsqrDs); mSolX = lsqr.Solution.GetWritableCopy(); for (int i = 0; i < layout.Length; i++) { layout[i].X = lsqr.Solution[i]; } for (int i = lsqrDs.Count - mKClust, j = 0; i < lsqrDs.Count; i++, j++) { lsqrDs[i].Label = mRefPos[j].Y; } lsqr.Train(lsqrDs); mSolY = lsqr.Solution.GetWritableCopy(); for (int i = 0; i < layout.Length; i++) { layout[i].Y = lsqr.Solution[i]; } return(settings == null ? layout : settings.AdjustLayout(layout)); }
public Vector2D[] ComputeLayout(LayoutSettings settings) { UnlabeledDataset <SparseVector <double> > dataset = new UnlabeledDataset <SparseVector <double> >(mDataset); // clustering mLogger.Info("ComputeLayout", "Clustering ..."); KMeansFast kMeans = new KMeansFast(mKClust); kMeans.Eps = mKMeansEps; kMeans.Random = mRandom; kMeans.Trials = 1; ClusteringResult clustering = kMeans.Cluster(mDataset); // throws ArgumentValueException // determine reference instances UnlabeledDataset <SparseVector <double> > dsRefInst = new UnlabeledDataset <SparseVector <double> >(); foreach (Cluster cluster in clustering.Roots) { SparseVector <double> centroid = cluster.Items.Count > 0 ? cluster.ComputeCentroid(mDataset, CentroidType.NrmL2) : new SparseVector <double>(); dsRefInst.Add(centroid); // dataset of reference instances dataset.Add(centroid); // add centroids to the main dataset } // position reference instances mLogger.Info("ComputeLayout", "Positioning reference instances ..."); SparseMatrix <double> simMtx = ModelUtils.GetDotProductSimilarity(dsRefInst, mSimThresh, /*fullMatrix=*/ false); StressMajorizationLayout sm = new StressMajorizationLayout(dsRefInst.Count, new DistFunc(simMtx)); sm.Random = mRandom; Vector2D[] centrPos = sm.ComputeLayout(); // k-NN mLogger.Info("ComputeLayout", "Computing similarities ..."); simMtx = ModelUtils.GetDotProductSimilarity(dataset, mSimThresh, /*fullMatrix=*/ true); mLogger.Info("ComputeLayout", "Constructing system of linear equations ..."); LabeledDataset <double, SparseVector <double> > lsqrDs = new LabeledDataset <double, SparseVector <double> >(); foreach (IdxDat <SparseVector <double> > simMtxRow in simMtx) { if (simMtxRow.Dat.Count <= 1) { mLogger.Warn("ComputeLayout", "Instance #{0} has no neighborhood.", simMtxRow.Idx); } ArrayList <KeyDat <double, int> > knn = new ArrayList <KeyDat <double, int> >(simMtxRow.Dat.Count); foreach (IdxDat <double> item in simMtxRow.Dat) { if (item.Idx != simMtxRow.Idx) { knn.Add(new KeyDat <double, int>(item.Dat, item.Idx)); } } knn.Sort(DescSort <KeyDat <double, int> > .Instance); int count = Math.Min(knn.Count, mKNN); SparseVector <double> eq = new SparseVector <double>(); double wgt = 1.0 / (double)count; for (int i = 0; i < count; i++) { eq.InnerIdx.Add(knn[i].Dat); eq.InnerDat.Add(-wgt); } eq.InnerIdx.Sort(); // *** sort only indices eq[simMtxRow.Idx] = 1; lsqrDs.Add(0, eq); } Vector2D[] layout = new Vector2D[dataset.Count - mKClust]; for (int i = dataset.Count - mKClust, j = 0; i < dataset.Count; i++, j++) { SparseVector <double> eq = new SparseVector <double>(new IdxDat <double>[] { new IdxDat <double>(i, 1) }); lsqrDs.Add(centrPos[j].X, eq); } LSqrModel lsqr = new LSqrModel(); lsqr.Train(lsqrDs); for (int i = 0; i < layout.Length; i++) { layout[i].X = lsqr.Solution[i]; } for (int i = lsqrDs.Count - mKClust, j = 0; i < lsqrDs.Count; i++, j++) { lsqrDs[i].Label = centrPos[j].Y; } lsqr.Train(lsqrDs); for (int i = 0; i < layout.Length; i++) { layout[i].Y = lsqr.Solution[i]; } return(settings == null ? layout : settings.AdjustLayout(layout)); }
// TODO: exceptions public Vector2D[] Update(int numDequeue, IEnumerable <SparseVector <double> > newInst, bool test, LayoutSettings settings, ref PtInfo[] ptInfo, int _count) { // clustering mLogger.Info("Update", "Clustering ..."); /*prof*/ StopWatch sw = new StopWatch(); mKMeans.Eps = mKMeansEps; int iter = 0; mKMeans.Update(numDequeue, newInst, ref iter); /*prof*/ sw.Save("cl.txt", _count, iter.ToString()); // determine reference instances /*prof*/ sw.Reset(); UnlabeledDataset <SparseVector <double> > dsRefInst = new UnlabeledDataset <SparseVector <double> >(); UnlabeledDataset <SparseVector <double> > dsNewInst = new UnlabeledDataset <SparseVector <double> >(newInst); foreach (SparseVector <double> centroid in mKMeans.GetCentroids()) { dsRefInst.Add(centroid); // dataset of reference instances dsNewInst.Add(centroid); // dataset of new instances } // position reference instances mLogger.Info("Update", "Positioning reference instances ..."); SparseMatrix <double> simMtx = ModelUtils.GetDotProductSimilarity(dsRefInst, mSimThresh, /*fullMatrix=*/ false); StressMajorizationLayout sm = new StressMajorizationLayout(dsRefInst.Count, new DistFunc(simMtx)); sm.Random = mRandom; sm.MaxSteps = int.MaxValue; sm.MinDiff = 1E-3; mRefPos = sm.ComputeLayout(/*settings=*/ null, mRefPos /*make this a property!!!*/); /*prof*/ sw.Save("sm.txt", _count); // k-NN /*prof*/ sw.Reset(); DateTime t = DateTime.Now; mLogger.Info("Update", "Computing similarities ..."); // update list of neighborhoods mPatches.RemoveRange(mDataset.Count - mKClust, mKClust); mPatches.RemoveRange(0, numDequeue); // remove instances from [dataset and] neighborhoods foreach (Patch patch in mPatches) { if (patch.Min != null && (patch.Min.Idx < numDequeue || patch.Max.Idx >= mDataset.Count - mKClust)) { int oldCount = patch.List.Count; ArrayList <KeyDat <double, Patch> > tmp = new ArrayList <KeyDat <double, Patch> >(); foreach (KeyDat <double, Patch> item in patch.List) { if (item.Dat.Idx >= numDequeue && item.Dat.Idx < mDataset.Count - mKClust) { tmp.Add(item); } //else //{ // Console.WriteLine("Remove {0}", item.Dat.Idx - numDequeue); //} } patch.List = tmp; patch.ProcessList(); patch.NeedUpdate = patch.List.Count < mKNn && oldCount >= mKNn; } } // update dataset mDataset.RemoveRange(mDataset.Count - mKClust, mKClust); mDataset.RemoveRange(0, numDequeue); // add new instances to dataset int preAddCount = mDataset.Count; mDataset.AddRange(dsNewInst); // precompute transposed matrices SparseMatrix <double> trNewInst = ModelUtils.GetTransposedMatrix(dsNewInst); SparseMatrix <double> trDataset = ModelUtils.GetTransposedMatrix(mDataset); // add new instances to neighborhoods for (int i = 0; i < dsNewInst.Count; i++) { mPatches.Add(new Patch(-1)); mPatches.Last.NeedUpdate = true; } for (int i = 0; i < mPatches.Count; i++) { mPatches[i].Idx = i; } for (int i = 0; i < mPatches.Count; i++) { Patch patch = mPatches[i]; SparseVector <double> vec = mDataset[i]; if (vec != null) { if (patch.NeedUpdate) // full update required { //if (i == 1347) { Console.WriteLine("full update"); } SparseVector <double> simVec = ModelUtils.GetDotProductSimilarity(trDataset, mDataset.Count, vec, mSimThresh); ArrayList <KeyDat <double, int> > tmp = new ArrayList <KeyDat <double, int> >(); foreach (IdxDat <double> item in simVec) { if (item.Idx != i) { tmp.Add(new KeyDat <double, int>(item.Dat, item.Idx)); } } tmp.Sort(new Comparer2()); int count = Math.Min(tmp.Count, mKNnExt); patch.List.Clear(); for (int j = 0; j < count; j++) { patch.List.Add(new KeyDat <double, Patch>(tmp[j].Key, mPatches[tmp[j].Dat])); } patch.ProcessList(); patch.NeedUpdate = false; } else // only new instances need to be considered { //if (i == 1347) { Console.WriteLine("partial update"); } SparseVector <double> simVec = ModelUtils.GetDotProductSimilarity(trNewInst, dsNewInst.Count, vec, mSimThresh); // check if further processing is needed bool needMerge = false; if (test) { foreach (IdxDat <double> item in simVec) { if (item.Dat >= patch.MinSim) { needMerge = true; //Console.WriteLine("{0} {1}", item.Dat, patch.MinSim); break; } } } else { foreach (IdxDat <double> item in simVec) { if (item.Dat > patch.MinSim) { needMerge = true; //Console.WriteLine("{0} {1}", item.Dat, patch.MinSim); break; } } } if (needMerge || patch.List.Count < mKNn) { //if (i == 1347) { Console.WriteLine("merge"); } int oldCount = patch.List.Count; ArrayList <KeyDat <double, Patch> > tmp = new ArrayList <KeyDat <double, Patch> >(); foreach (IdxDat <double> item in simVec) { tmp.Add(new KeyDat <double, Patch>(item.Dat, mPatches[item.Idx + preAddCount])); } // merge the two lists // TODO: speed this up patch.List.AddRange(tmp); patch.List.Sort(new Comparer()); // trim list to size if (oldCount >= mKNn) { patch.List.RemoveRange(oldCount, patch.List.Count - oldCount); } patch.ProcessList(); } } } } /*prof*/ sw.Save("knn.txt", _count); // *** Test *** sw.Reset(); ModelUtils.GetDotProductSimilarity(mDataset, mSimThresh, /*fullMatrix=*/ true); sw.Save("selfSim.txt", _count, mDataset.Count.ToString()); if (test) { simMtx = ModelUtils.GetDotProductSimilarity(mDataset, mSimThresh, /*fullMatrix=*/ true); ArrayList <Patch> patches = new ArrayList <Patch>(); for (int i = 0; i < mDataset.Count; i++) { patches.Add(new Patch(i)); } foreach (IdxDat <SparseVector <double> > simMtxRow in simMtx) { if (simMtxRow.Dat.Count <= 1) { mLogger.Warn("Update", "Instance #{0} has no neighborhood.", simMtxRow.Idx); } ArrayList <KeyDat <double, int> > knn = new ArrayList <KeyDat <double, int> >(simMtxRow.Dat.Count); foreach (IdxDat <double> item in simMtxRow.Dat) { if (item.Idx != simMtxRow.Idx) { knn.Add(new KeyDat <double, int>(item.Dat, item.Idx)); } } knn.Sort(new Comparer2()); int count = Math.Min(knn.Count, mKNnExt); for (int i = 0; i < count; i++) { patches[simMtxRow.Idx].List.Add(new KeyDat <double, Patch>(knn[i].Key, patches[knn[i].Dat])); } patches[simMtxRow.Idx].ProcessList(); } // compare if (patches.Count != mPatches.Count) { throw new Exception("Count mismatch."); } for (int i = 0; i < mPatches.Count; i++) { if (patches[i].List.Count < mKNn && patches[i].List.Count != mPatches[i].List.Count) { Console.WriteLine(mPatches[i].List.Count); Console.WriteLine(patches[i].List.Count); Output(mPatches[i].List); Output(patches[i].List); Console.WriteLine(i); throw new Exception("List count mismatch."); } int count = Math.Min(mPatches[i].List.Count, mKNn); for (int j = 0; j < count; j++) { //Console.WriteLine("{4} {0}-{1} {2}-{3}", mPatches[i].List[j].Key, mPatches[i].List[j].Dat.Idx, patches[i].List[j].Key, patches[i].List[j].Dat.Idx, i); if (mPatches[i].List[j].Key != patches[i].List[j].Key || mPatches[i].List[j].Dat.Idx != patches[i].List[j].Dat.Idx) { Console.WriteLine("i:{4} fast:{0}-{1} slow:{2}-{3}", mPatches[i].List[j].Key, mPatches[i].List[j].Dat.Idx, patches[i].List[j].Key, patches[i].List[j].Dat.Idx, i); int idxFast = mPatches[i].List[j].Dat.Idx; int idxSlow = patches[i].List[j].Dat.Idx; Console.WriteLine("slow @ fast idx: {0}", GetKey(patches[i].List, idxFast)); Console.WriteLine("fast @ slow idx: {0}", GetKey(mPatches[i].List, idxSlow)); throw new Exception("Patch item mismatch."); } } } } // *** End of test *** //Console.WriteLine("Number of patches: {0}", mPatches.Count); //int waka = 0; //foreach (Patch patch in mPatches) //{ // waka += patch.List.Count; //} //Console.WriteLine("Avg list size: {0}", (double)waka / (double)mPatches.Count); Console.WriteLine((DateTime.Now - t).TotalMilliseconds); /*prof*/ sw.Reset(); mLogger.Info("Update", "Constructing system of linear equations ..."); LabeledDataset <double, SparseVector <double> > lsqrDs = new LabeledDataset <double, SparseVector <double> >(); Vector2D[] layout = new Vector2D[mDataset.Count - mKClust]; foreach (Patch patch in mPatches) { int count = Math.Min(patch.List.Count, mKNn); SparseVector <double> eq = new SparseVector <double>(); double wgt = 1.0 / (double)count; for (int i = 0; i < count; i++) { eq.InnerIdx.Add(patch.List[i].Dat.Idx); eq.InnerDat.Add(-wgt); } eq.InnerIdx.Sort(); // *** sort only indices eq[patch.Idx] = 1; lsqrDs.Add(0, eq); } for (int i = mDataset.Count - mKClust, j = 0; i < mDataset.Count; i++, j++) { SparseVector <double> eq = new SparseVector <double>(new IdxDat <double>[] { new IdxDat <double>(i, 1) }); lsqrDs.Add(mRefPos[j].X, eq); } LSqrModel lsqr = new LSqrModel(); mSolX.RemoveRange(0, numDequeue); double[] aux = new double[mKClust]; mSolX.CopyTo(mSolX.Count - mKClust, aux, 0, mKClust); mSolX.RemoveRange(mSolX.Count - mKClust, mKClust); foreach (SparseVector <double> newVec in newInst) { mSolX.Add(0); } mSolX.AddRange(aux); lsqr.InitialSolution = mSolX.ToArray(); lsqr.Train(lsqrDs); mSolX = lsqr.Solution.GetWritableCopy(); //for (int i = 0; i < lsqr.InitialSolution.Length; i++) //{ // Console.WriteLine("{0}\t{1}", lsqr.InitialSolution[i], lsqr.Solution[i]); //} for (int i = 0; i < layout.Length; i++) { layout[i].X = lsqr.Solution[i]; } for (int i = lsqrDs.Count - mKClust, j = 0; i < lsqrDs.Count; i++, j++) { lsqrDs[i].Label = mRefPos[j].Y; } mSolY.RemoveRange(0, numDequeue); aux = new double[mKClust]; mSolY.CopyTo(mSolY.Count - mKClust, aux, 0, mKClust); mSolY.RemoveRange(mSolY.Count - mKClust, mKClust); foreach (SparseVector <double> newVec in newInst) { mSolY.Add(0); } mSolY.AddRange(aux); lsqr.InitialSolution = mSolY.ToArray(); lsqr.Train(lsqrDs); mSolY = lsqr.Solution.GetWritableCopy(); for (int i = 0; i < layout.Length; i++) { layout[i].Y = lsqr.Solution[i]; } /*prof*/ sw.Save("lsqr.txt", _count); // ----------------------------------------------------------------- // make ptInfo // ----------------------------------------------------------------- ptInfo = new PtInfo[layout.Length]; int ii = 0; foreach (Vector2D pt in layout) { ptInfo[ii] = new PtInfo(); ptInfo[ii].X = pt.X; ptInfo[ii].Y = pt.Y; ptInfo[ii].Vec = mDataset[ii]; ii++; } // ----------------------------------------------------------------- return(settings == null ? layout : settings.AdjustLayout(layout)); }