/// <summary> /// Trains a POS tag maximum entropy model /// </summary> /// <param name="trainingFile">filepath to the training data</param> /// <param name="iterations">number of training iterations to perform</param> /// <param name="cutoff">Cutoff value to use for the data indexer</param> /// <returns>Trained GIS model</returns> public static SharpEntropy.GisModel TrainModel(string trainingFile, int iterations, int cutoff) { #if DNF SharpEntropy.ITrainingEventReader eventReader = new PosEventReader(new StreamReader(trainingFile)); #else SharpEntropy.ITrainingEventReader eventReader = new PosEventReader(new StreamReader(new FileStream(trainingFile, FileMode.OpenOrCreate, FileAccess.Read))); #endif return(Train(eventReader, iterations, cutoff)); }
public virtual void LocalEvaluate(SharpEntropy.IMaximumEntropyModel posModel, StreamReader reader, out double accuracy, out double sentenceAccuracy) { this.PosModel = posModel; float total = 0, correct = 0, sentences = 0, sentencesCorrect = 0; var sentenceReader = new StreamReader(reader.BaseStream, System.Text.Encoding.UTF7); string line; while ((object)(line = sentenceReader.ReadLine()) != null) { sentences++; var annotatedPair = PosEventReader.ConvertAnnotatedString(line); var words = annotatedPair.Item1; var outcomes = annotatedPair.Item2; var tags = new ArrayList(Beam.BestSequence(words.ToArray(), null).Outcomes); int count = 0; bool isSentenceOk = true; for (IEnumerator tagIndex = tags.GetEnumerator(); tagIndex.MoveNext(); count++) { total++; var tag = (string)tagIndex.Current; if (tag == (string)outcomes[count]) { correct++; } else { isSentenceOk = false; } } if (isSentenceOk) { sentencesCorrect++; } } accuracy = correct / total; sentenceAccuracy = sentencesCorrect / sentences; }
public virtual void LocalEvaluate(SharpEntropy.IMaximumEntropyModel posModel, System.IO.StreamReader reader, out double accuracy, out double sentenceAccuracy) { mPosModel = posModel; float total = 0, correct = 0, sentences = 0, sentencesCorrect = 0; System.IO.StreamReader sentenceReader = new System.IO.StreamReader(reader.BaseStream, System.Text.Encoding.UTF7); string line; while ((object)(line = sentenceReader.ReadLine()) != null) { sentences++; Util.Pair <ArrayList, ArrayList> annotatedPair = PosEventReader.ConvertAnnotatedString(line); ArrayList words = annotatedPair.FirstValue; ArrayList outcomes = annotatedPair.SecondValue; ArrayList tags = new ArrayList(Beam.BestSequence(words, null).Outcomes); int count = 0; bool isSentenceOK = true; for (System.Collections.IEnumerator tagIndex = tags.GetEnumerator(); tagIndex.MoveNext(); count++) { total++; string tag = (string)tagIndex.Current; if (tag == (string)outcomes[count]) { correct++; } else { isSentenceOK = false; } } if (isSentenceOK) { sentencesCorrect++; } } accuracy = correct / total; sentenceAccuracy = sentencesCorrect / sentences; }
/// <summary> /// Trains a POS tag maximum entropy model. /// </summary> /// <param name="trainingFile"> /// filepath to the training data. /// </param> /// <param name="iterations"> /// number of training iterations to perform. /// </param> /// <param name="cutoff"> /// Cutoff value to use for the data indexer. /// </param> /// <returns> /// Trained GIS model. /// </returns> public static SharpEntropy.GisModel TrainModel(string trainingFile, int iterations, int cutoff) { SharpEntropy.ITrainingEventReader eventReader = new PosEventReader(new System.IO.StreamReader(trainingFile)); return Train(eventReader, iterations, cutoff); }
/// <summary> /// Trains a POS tag maximum entropy model /// </summary> /// <param name="trainingFile">filepath to the training data</param> /// <param name="iterations">number of training iterations to perform</param> /// <param name="cutoff">Cutoff value to use for the data indexer</param> /// <returns>Trained GIS model</returns> public static SharpEntropy.GisModel TrainModel(string trainingFile, int iterations, int cutoff) { SharpEntropy.ITrainingEventReader eventReader = new PosEventReader(new StreamReader(trainingFile)); return(Train(eventReader, iterations, cutoff)); }