public double getTotalPossibleScore(NaturalLanguageData target, string botName, double tripletMatchValue) { //TODO: exclude names, or make them less weight var score = 0.0; foreach (var sentence in target.sentences) { if (sentence.triplets.subject != null) { score += tripletMatchValue; } else { foreach (var token in sentence.tokens) { var tagValue = NaturalLanguageService.NaturalLanguageService.POSTagValues[token.POSTag]; if (tagValue != null && token.Lexeme.ToLower() != botName.ToLower() && token.Lexeme.ToLower() != "@" + botName.ToLower()) { score += (double)tagValue + ((double)tagValue * .1); } } } } return(score); }
private double GetNaturalLanguageDataScore(NaturalLanguageData target, NaturalLanguageData existing) { var sentencesScore = 0.0; if (target.sentences == null || existing.sentences == null) { return(0); } foreach (var targetSentence in target.sentences) { var bestScore = 0.0; foreach (var existingSentence in existing.sentences) { var score = _sentenceScoreService.GetScore(targetSentence, existingSentence); if (score > bestScore) { bestScore = score; } } sentencesScore += bestScore; } var maxSentencesScore = (double)target.sentences.Count; if (maxSentencesScore == 0) { return(0); } return(sentencesScore / maxSentencesScore); }
public NaturalLanguageData AnalyzeMessage(Chat chat) { var document = new Document(chat.message); var sentences = new List <ChatModels.Sentence>(); foreach (var s in document.Sentences) { var sentence = new ChatModels.Sentence(); sentence.tokens = new List <Token>(); for (var index = 0; index < s.Words.Count; index++) { var token = new Token(); token.POSTag = s.PosTags.ToList()[index]; token.Lexeme = s.Lemmas.ToList()[index]; token.Stem = s.Lemmas.ToList()[index]; sentence.tokens.Add(token); } sentence.interrogative = isInterrogative(s); sentence.chunks = new List <Chunk>(); sentence.triplets = new Triplets(); sentences.Add(sentence); } var naturalLanguageData = new NaturalLanguageData(); naturalLanguageData.sentences = sentences; return(naturalLanguageData); }
public double GetMatchConfidence(NaturalLanguageData target, NaturalLanguageData existing, string botName) { var confidence = GetNaturalLanguageDataScore(target, existing); var reverseConfidence = GetNaturalLanguageDataScore(existing, target); var score = (confidence + reverseConfidence) / 2.0; return(score); }
public double getMatchConfidence(NaturalLanguageData target, NaturalLanguageData naturalLanguageDocument, string botName) { var confidence = getOneWayMatchConfidence(target, naturalLanguageDocument, botName); var reverseConfidence = getOneWayMatchConfidence(naturalLanguageDocument, target, botName); var score = (confidence + reverseConfidence) / 2.0; return(score); }
public double getInterrogativeScore(NaturalLanguageData target, NaturalLanguageData naturalLanguageDocument) { if (target.sentences.Any(s => s.interrogative) != naturalLanguageDocument.sentences.Any(s => s.interrogative)) { return(.5); } return(1); }
public static NaturalLanguageData AnalyzeMessage(Chat chat) { var document = new Document("en", chat.message); var sentences = new List <Sentence>(); try { analyzer.Analyze(document); foreach (var s in document.Sentences) { var sentence = new Sentence(); sentence.tokens = new List <Token>(); foreach (var t in s.Tokens) { var token = new Token(); token.POSTag = t.POSTag; token.Lexeme = t.Lexeme; token.Stem = wordStemmer.GetSteamWord(t.Lexeme); sentence.tokens.Add(token); } sentence.chunks = new List <Chunk>(); foreach (var c in s.Chunks) { var chunk = new Chunk(); chunk.tag = c.Tag; chunk.tokens = new List <Token>(); foreach (var t in c.Tokens) { var token = new Token(); token.POSTag = t.POSTag; token.Lexeme = t.Lexeme; token.Stem = wordStemmer.GetSteamWord(t.Lexeme); chunk.tokens.Add(token); } sentence.chunks.Add(chunk); } sentence.interrogative = isInterrogative(s); sentence.triplets = tripletService.GetSentenceTriplets(sentence); sentences.Add(sentence); } } catch (AnalyzerException) { } var naturalLanguageData = new NaturalLanguageData(); naturalLanguageData.sentences = sentences; return(naturalLanguageData); }
public double getOneWayMatchConfidence(NaturalLanguageData target, NaturalLanguageData naturalLanguageDocument, string botName) { //TODO: exclude names, match them, or make them less weight var sentencesScore = 0.0; foreach (var targetSentence in target.sentences) { sentencesScore += getSentenceScore(targetSentence, naturalLanguageDocument, botName); } var maxSentencesScore = target.sentences.Count; var score = sentencesScore / maxSentencesScore; score = score * interogativeScoreService.getInterrogativeScore(target, naturalLanguageDocument); return(score); }
public double getSentenceScore(Sentence targetSentence, NaturalLanguageData naturalLanguageDocument, string botName) { if (targetSentence.triplets.subject != null) { return(tripletScoreService.getBestTripletScore(targetSentence.triplets, naturalLanguageDocument.sentences)); } else { var score = 0.0; var maximumScore = 0.0; foreach (var targetToken in targetSentence.tokens) { score += tokenScoreService.getTokenScore(targetToken, naturalLanguageDocument.sentences); maximumScore += tokenScoreService.getTokenValue(targetToken); } return(score / maximumScore); } }