public double getTotalPossibleScore(NaturalLanguageData target, string botName, double tripletMatchValue)
{
//TODO: exclude names, or make them less weight
var score = 0.0;
foreach (var sentence in target.sentences)
{
if (sentence.triplets.subject != null)
{
score += tripletMatchValue;
}
else
{
foreach (var token in sentence.tokens)
{
var tagValue = NaturalLanguageService.NaturalLanguageService.POSTagValues[token.POSTag];
if (tagValue != null && token.Lexeme.ToLower() != botName.ToLower() && token.Lexeme.ToLower() != "@" + botName.ToLower())
{
score += (double)tagValue + ((double)tagValue * .1);
}
}
}
}
return(score);
}
private double GetNaturalLanguageDataScore(NaturalLanguageData target, NaturalLanguageData existing)
{
var sentencesScore = 0.0;
if (target.sentences == null || existing.sentences == null)
{
return(0);
}
foreach (var targetSentence in target.sentences)
{
var bestScore = 0.0;
foreach (var existingSentence in existing.sentences)
{
var score = _sentenceScoreService.GetScore(targetSentence, existingSentence);
if (score > bestScore)
{
bestScore = score;
}
}
sentencesScore += bestScore;
}
var maxSentencesScore = (double)target.sentences.Count;
if (maxSentencesScore == 0)
{
return(0);
}
return(sentencesScore / maxSentencesScore);
}
public NaturalLanguageData AnalyzeMessage(Chat chat)
{
var document = new Document(chat.message);
var sentences = new List <ChatModels.Sentence>();
foreach (var s in document.Sentences)
{
var sentence = new ChatModels.Sentence();
sentence.tokens = new List <Token>();
for (var index = 0; index < s.Words.Count; index++)
{
var token = new Token();
token.POSTag = s.PosTags.ToList()[index];
token.Lexeme = s.Lemmas.ToList()[index];
token.Stem = s.Lemmas.ToList()[index];
sentence.tokens.Add(token);
}
sentence.interrogative = isInterrogative(s);
sentence.chunks = new List <Chunk>();
sentence.triplets = new Triplets();
sentences.Add(sentence);
}
var naturalLanguageData = new NaturalLanguageData();
naturalLanguageData.sentences = sentences;
return(naturalLanguageData);
}
public double GetMatchConfidence(NaturalLanguageData target, NaturalLanguageData existing, string botName)
{
var confidence = GetNaturalLanguageDataScore(target, existing);
var reverseConfidence = GetNaturalLanguageDataScore(existing, target);
var score = (confidence + reverseConfidence) / 2.0;
return(score);
}
public double getMatchConfidence(NaturalLanguageData target, NaturalLanguageData naturalLanguageDocument, string botName)
{
var confidence = getOneWayMatchConfidence(target, naturalLanguageDocument, botName);
var reverseConfidence = getOneWayMatchConfidence(naturalLanguageDocument, target, botName);
var score = (confidence + reverseConfidence) / 2.0;
return(score);
}
public double getInterrogativeScore(NaturalLanguageData target, NaturalLanguageData naturalLanguageDocument)
{
if (target.sentences.Any(s => s.interrogative) != naturalLanguageDocument.sentences.Any(s => s.interrogative))
{
return(.5);
}
return(1);
}
public static NaturalLanguageData AnalyzeMessage(Chat chat)
{
var document = new Document("en", chat.message);
var sentences = new List <Sentence>();
try
{
analyzer.Analyze(document);
foreach (var s in document.Sentences)
{
var sentence = new Sentence();
sentence.tokens = new List <Token>();
foreach (var t in s.Tokens)
{
var token = new Token();
token.POSTag = t.POSTag;
token.Lexeme = t.Lexeme;
token.Stem = wordStemmer.GetSteamWord(t.Lexeme);
sentence.tokens.Add(token);
}
sentence.chunks = new List <Chunk>();
foreach (var c in s.Chunks)
{
var chunk = new Chunk();
chunk.tag = c.Tag;
chunk.tokens = new List <Token>();
foreach (var t in c.Tokens)
{
var token = new Token();
token.POSTag = t.POSTag;
token.Lexeme = t.Lexeme;
token.Stem = wordStemmer.GetSteamWord(t.Lexeme);
chunk.tokens.Add(token);
}
sentence.chunks.Add(chunk);
}
sentence.interrogative = isInterrogative(s);
sentence.triplets = tripletService.GetSentenceTriplets(sentence);
sentences.Add(sentence);
}
}
catch (AnalyzerException)
{
}
var naturalLanguageData = new NaturalLanguageData();
naturalLanguageData.sentences = sentences;
return(naturalLanguageData);
}
public double getOneWayMatchConfidence(NaturalLanguageData target, NaturalLanguageData naturalLanguageDocument, string botName)
{
//TODO: exclude names, match them, or make them less weight
var sentencesScore = 0.0;
foreach (var targetSentence in target.sentences)
{
sentencesScore += getSentenceScore(targetSentence, naturalLanguageDocument, botName);
}
var maxSentencesScore = target.sentences.Count;
var score = sentencesScore / maxSentencesScore;
score = score * interogativeScoreService.getInterrogativeScore(target, naturalLanguageDocument);
return(score);
}
public double getSentenceScore(Sentence targetSentence, NaturalLanguageData naturalLanguageDocument, string botName)
{
if (targetSentence.triplets.subject != null)
{
return(tripletScoreService.getBestTripletScore(targetSentence.triplets, naturalLanguageDocument.sentences));
}
else
{
var score = 0.0;
var maximumScore = 0.0;
foreach (var targetToken in targetSentence.tokens)
{
score += tokenScoreService.getTokenScore(targetToken, naturalLanguageDocument.sentences);
maximumScore += tokenScoreService.getTokenValue(targetToken);
}
return(score / maximumScore);
}
}