/// <summary> /// Tokenizes the string. /// </summary> /// <param name="input"> /// The string to be tokenized. /// </param> /// <returns> /// A span array containing individual tokens as elements. /// </returns> public virtual Util.Span[] TokenizePositions(string input) { if (mUnicodeMapping) { input = Utils.MapUnicodeChars(input); } Util.Span[] tokens = Split(input); mNewTokens.Clear(); mTokenProbabilities.Clear(); for (int currentToken = 0, tokenCount = tokens.Length; currentToken < tokenCount; currentToken++) { Util.Span tokenSpan = tokens[currentToken]; string token = input.Substring(tokenSpan.Start, (tokenSpan.End) - (tokenSpan.Start)); // Can't tokenize single characters if (token.Length < 2) { mNewTokens.Add(tokenSpan); mTokenProbabilities.Add(1.0); } else if (AlphaNumericOptimization && AlphaNumeric.IsMatch(token)) { mNewTokens.Add(tokenSpan); mTokenProbabilities.Add(1.0); } else { int startPosition = tokenSpan.Start; int endPosition = tokenSpan.End; int originalStart = tokenSpan.Start; double tokenProbability = 1.0; for (int currentPosition = originalStart + 1; currentPosition < endPosition; currentPosition++) { //Console.Write("{0} {1}|{2} ({3})", currentPosition - originalStart - 1, token[currentPosition - originalStart - 1], token[currentPosition - originalStart], token); if (mAlphaNumericOptimization) { char leftChar = token[currentPosition - originalStart - 1]; char rightChar = token[currentPosition - originalStart]; if (char.IsLetterOrDigit(leftChar) && char.IsLetterOrDigit(rightChar)) /*Console.WriteLine();*/ continue { ; } } double[] probabilities = mModel.Evaluate(mContextGenerator.GetContext(new Util.Pair <string, int>(token, currentPosition - originalStart))); string bestOutcome = mModel.GetBestOutcome(probabilities); //Console.WriteLine(bestOutcome); tokenProbability *= probabilities[mModel.GetOutcomeIndex(bestOutcome)]; if (bestOutcome == TokenContextGenerator.SplitIndicator) { mNewTokens.Add(new Util.Span(startPosition, currentPosition)); mTokenProbabilities.Add(tokenProbability); startPosition = currentPosition; tokenProbability = 1.0; } } mNewTokens.Add(new Util.Span(startPosition, endPosition)); mTokenProbabilities.Add(tokenProbability); } }
/// <summary> /// Tokenizes the string. /// </summary> /// <param name="input"> /// The string to be tokenized. /// </param> /// <returns> /// A span array containing individual tokens as elements. /// </returns> public virtual Util.Span[] TokenizePositions(string input) { Util.Span[] tokens = Split(input); mNewTokens.Clear(); mTokenProbabilities.Clear(); for (int currentToken = 0, tokenCount = tokens.Length; currentToken < tokenCount; currentToken++) { Util.Span tokenSpan = tokens[currentToken]; string token = input.Substring(tokenSpan.Start, (tokenSpan.End) - (tokenSpan.Start)); // Can't tokenize single characters if (token.Length < 2) { mNewTokens.Add(tokenSpan); mTokenProbabilities.Add(1.0); } else if (AlphaNumericOptimization && AlphaNumeric.IsMatch(token)) { mNewTokens.Add(tokenSpan); mTokenProbabilities.Add(1.0); } else { int startPosition = tokenSpan.Start; int endPosition = tokenSpan.End; int originalStart = tokenSpan.Start; double tokenProbability = 1.0; for (int currentPosition = originalStart + 1; currentPosition < endPosition; currentPosition++) { double[] probabilities = mModel.Evaluate(mContextGenerator.GetContext(new Util.Pair <string, int>(token, currentPosition - originalStart))); string bestOutcome = mModel.GetBestOutcome(probabilities); tokenProbability *= probabilities[mModel.GetOutcomeIndex(bestOutcome)]; if (bestOutcome == TokenContextGenerator.SplitIndicator) { mNewTokens.Add(new Util.Span(startPosition, currentPosition)); mTokenProbabilities.Add(tokenProbability); startPosition = currentPosition; tokenProbability = 1.0; } } mNewTokens.Add(new Util.Span(startPosition, endPosition)); mTokenProbabilities.Add(tokenProbability); } } return(mNewTokens.ToArray()); }
/// <summary> /// Detect the position of the first words of sentences in a string. /// </summary> /// <param name="input"> /// The string to be processed. /// </param> /// <returns> /// A integer array containing the positions of the end index of /// every sentence /// </returns> public virtual int[] SentencePositionDetect(string input) { if (mUnicodeMapping) { input = Utils.MapUnicodeChars(input); } double sentenceProbability = 1; mSentenceProbs.Clear(); System.Text.StringBuilder buffer = new System.Text.StringBuilder(input); List <int> endersList = mScanner.GetPositions(input); List <int> positions = new List <int>(endersList.Count); for (int currentEnder = 0, enderCount = endersList.Count, index = 0; currentEnder < enderCount; currentEnder++) { int candidate = endersList[currentEnder]; int cInt = candidate; // skip over the leading parts of non-token final delimiters int firstWhiteSpace = GetFirstWhitespace(input, cInt + 1); if (((currentEnder + 1) < enderCount) && ((endersList[currentEnder + 1]) < firstWhiteSpace)) { continue; } Util.Pair <System.Text.StringBuilder, int> pair = new Util.Pair <System.Text.StringBuilder, int>(buffer, candidate); double[] probabilities = mModel.Evaluate(mContextGenerator.GetContext(pair)); string bestOutcome = mModel.GetBestOutcome(probabilities); sentenceProbability *= probabilities[mModel.GetOutcomeIndex(bestOutcome)]; if (bestOutcome.Equals("T") && IsAcceptableBreak(input, index, cInt)) { if (index != cInt) { positions.Add(GetFirstNonWhitespace(input, GetFirstWhitespace(input, cInt + 1))); //moIntegerPool.GetInteger(GetFirstNonWhitespace(input, GetFirstWhitespace(input, cInt + 1)))); mSentenceProbs.Add(probabilities[mModel.GetOutcomeIndex(bestOutcome)]); } index = cInt + 1; } } return(positions.ToArray()); }
// Methods -------------------- private void AddEvents(string line) { string[] wordsWithSeparatorToken = line.Split(' '); foreach (string wordWithSeparatorToken in wordsWithSeparatorToken) { var parts = wordWithSeparatorToken.Split(_tokenSeparator); var indicesOfSeparators = new List <int>(); for (var i = 1; i < parts.Length; i++) { var indexOfSeparator = parts.Where((p, index) => index < i).Sum(p => p.Length); indicesOfSeparators.Add(indexOfSeparator); } var word = string.Join("", parts); for (int index = 0; index < word.Length; index++) { string[] context = ContextGenerator.GetContext(new Tuple <string, int>(word, index)); var outcome = indicesOfSeparators.Contains(index) ? "T" : "F"; var trainingEvent = new SharpEntropy.TrainingEvent(outcome, context); _eventList.Add(trainingEvent); } } }
private void AddEvents(string line) { string[] spacedTokens = line.Split(' '); for (int currentToken = 0; currentToken < spacedTokens.Length; currentToken++) { string buffer = spacedTokens[currentToken]; if (MaximumEntropyTokenizer.AlphaNumeric.IsMatch(buffer)) { int lastIndex = buffer.Length - 1; for (int index = 0; index < buffer.Length; index++) { string[] context = mContextGenerator.GetContext(new Util.Pair <string, int>(buffer, index)); if (index == lastIndex) { mEventList.Add(new SharpEntropy.TrainingEvent("T", context)); } else { mEventList.Add(new SharpEntropy.TrainingEvent("F", context)); } } } } }
private void AddNewEvents(string token) { System.Text.StringBuilder buffer = mBuffer; buffer.Append(token.Trim()); int sentenceEndPosition = buffer.Length - 1; //add following word to stringbuilder if (mNext != null && token.Length > 0) { int positionAfterFirstWordInNext = mNext.IndexOf(" "); if (positionAfterFirstWordInNext != -1) { // should maybe changes this so that it usually adds a space // before the next sentence, but sometimes leaves no space. buffer.Append(" "); buffer.Append(mNext.Substring(0, (positionAfterFirstWordInNext) - (0))); } } for (System.Collections.IEnumerator iterator = mScanner.GetPositions(buffer).GetEnumerator(); iterator.MoveNext();) { int candidate = (int)iterator.Current; Util.Pair <System.Text.StringBuilder, int> pair = new Util.Pair <System.Text.StringBuilder, int>(buffer, candidate); string type = (candidate == sentenceEndPosition) ? "T" : "F"; SentenceDetectionEvent sentenceEvent = new SentenceDetectionEvent(type, mContextGenerator.GetContext(pair)); if (null != mTail) { mTail.NextEvent = sentenceEvent; mTail = sentenceEvent; } else if (null == mHead) { mHead = sentenceEvent; } else if (null == mHead.NextEvent) { mHead.NextEvent = mTail = sentenceEvent; } } buffer.Length = 0; }
public virtual void AddEvents(Util.Span[] tokens, string input) { if (tokens.Length > 0) { int startPosition = tokens[0].Start; int endPosition = tokens[tokens.Length - 1].End; string sentence = input.Substring(startPosition, (endPosition) - (startPosition)); Util.Span[] candidateTokens = MaximumEntropyTokenizer.SplitOnWhitespaces(sentence); int firstTrainingToken = -1; int lastTrainingToken = -1; for (int currentCandidate = 0; currentCandidate < candidateTokens.Length; currentCandidate++) { Util.Span candidateSpan = candidateTokens[currentCandidate]; string candidateToken = sentence.Substring(candidateSpan.Start, (candidateSpan.End) - (candidateSpan.Start)); //adjust candidateSpan to text offsets candidateSpan = new Util.Span(candidateSpan.Start + startPosition, candidateSpan.End + startPosition); //should we skip this token if (candidateToken.Length > 1 && (!mSkipAlphanumerics || !MaximumEntropyTokenizer.AlphaNumeric.IsMatch(candidateToken))) { //find offsets of annotated tokens inside candidate tokens bool foundTrainingTokens = false; for (int currentToken = lastTrainingToken + 1; currentToken < tokens.Length; currentToken++) { if (candidateSpan.Contains(tokens[currentToken])) { if (!foundTrainingTokens) { firstTrainingToken = currentToken; foundTrainingTokens = true; } lastTrainingToken = currentToken; } else if (candidateSpan.End < tokens[currentToken].End) { break; } else if (tokens[currentToken].End < candidateSpan.Start) { //keep looking } else { throw new ApplicationException("Bad training token: " + tokens[currentToken] + " cand: " + candidateSpan); } } // create training data if (foundTrainingTokens) { for (int currentToken = firstTrainingToken; currentToken <= lastTrainingToken; currentToken++) { Util.Span trainingTokenSpan = tokens[currentToken]; int candidateStart = candidateSpan.Start; for (int currentPosition = trainingTokenSpan.Start + 1; currentPosition < trainingTokenSpan.End; currentPosition++) { string[] context = mContextGenerator.GetContext(new Tuple <string, int>(candidateToken, currentPosition - candidateStart)); mEvents.Add(new SharpEntropy.TrainingEvent(TokenContextGenerator.NoSplitIndicator, context)); } if (trainingTokenSpan.End != candidateSpan.End) { string[] context = mContextGenerator.GetContext(new Tuple <string, int>(candidateToken, trainingTokenSpan.End - candidateStart)); mEvents.Add(new SharpEntropy.TrainingEvent(TokenContextGenerator.SplitIndicator, context)); } } } } } } }