public List <ConversationSubject> GetSubjects(AnalyzedChat response)
{
var subjects = new List <ConversationSubject>();
if (response.naturalLanguageData.sentences == null)
{
return(subjects);
}
foreach (var sentence in response.naturalLanguageData.sentences)
{
var token = sentence.Subject;
if (token != null && !string.IsNullOrWhiteSpace(token.Lemmas))
{
var index = subjects.FindIndex(s => s.Lemmas == token.Lemmas);
if (index >= 0)
{
subjects[index].OccurenceCount++;
}
else
{
var subject = new ConversationSubject
{
OccurenceCount = 1,
Lemmas = token.Lemmas
};
subjects.Add(subject);
}
}
}
return(subjects);
}
public void GetMultipleSubjects()
{
var sentences = new List <Sentence>();
sentences.Add(new Sentence {
Subject = new Token {
Lemmas = "ninja"
}
});
sentences.Add(new Sentence {
Subject = new Token {
Lemmas = "people"
}
});
var analyzedChat = new AnalyzedChat
{
naturalLanguageData = new NaturalLanguageData {
sentences = sentences
}
};
var service = new ResponseSubjectService();
var result = service.GetSubjects(analyzedChat);
Assert.Equal(2, result.Count);
}
private ChatResponse GetTripletDefinition(AnalyzedChat request)
{
if (request.naturalLanguageData.sentences[0].triplets.objectTriplet != null)
{
var definitions = new List <ChatResponse>();
foreach (var targetToken in request.naturalLanguageData.sentences[0].triplets.objectTriplet.chunk.tokens)
{
if (targetToken.POSTag == "NN" || targetToken.POSTag == "NNS" || targetToken.POSTag == "NNP" || targetToken.POSTag == "NNPS" || targetToken.POSTag == "VBG")
{
var definition = GetDefinition(targetToken.Lexeme);
if (definition.confidence > 0)
{
definitions.Add(definition);
}
}
}
if (definitions.Count() > 0)
{
var bestDefinition = definitions.OrderByDescending(x => x.confidence).First();
bestDefinition.confidence = urbanDictionaryTokenScore;
return(bestDefinition);
}
}
return(new ChatResponse()
{
response = new List <string>(), confidence = 0
});
}
public void GetProximitySubjects()
{
var responses = new List <AnalyzedChat>();
var sentences = new List <Sentence>();
sentences.Add(new Sentence {
Subject = new Token {
Word = "ninjas", Lemmas = "ninja"
}
});
var analyzedChat = new AnalyzedChat
{
naturalLanguageData = new NaturalLanguageData {
sentences = sentences
}
};
responses.Add(analyzedChat);
analyzedChat = new AnalyzedChat
{
naturalLanguageData = new NaturalLanguageData {
sentences = sentences
}
};
responses.Add(analyzedChat);
var conversation = new Conversation();
conversation.responses = responses;
var service = new ConversationSubjectService(new ResponseSubjectService());
var result = service.GetProximitySubjects(conversation, 0);
Assert.Single(result);
}
public bool UpdateDatabases(ConversationRequest conversationRequest)
{
if (_conversationService.ConversationLength(conversationRequest.name, conversationRequest.type) >= conversationRequest.responses.Count())
{
return(false);
}
var conversation = new Conversation
{
name = conversationRequest.name,
responses = conversationRequest.responses
};
var analyzedConversation = _analyzationService.AnalyzeConversationAsync(conversation);
var conversationUdpdated = _covnersationUpdateService.UpdateConversation(analyzedConversation, conversationRequest.type);
AnalyzedChat previousChat = null;
foreach (var analyziedChat in analyzedConversation.responses)
{
_userService.UpdateUsers(analyziedChat, previousChat);
previousChat = analyziedChat;
}
return(conversationUdpdated);
}
public void UpdateUsers(AnalyzedChat userResponse, AnalyzedChat question)
{
var nickName = userNickNameService.GetNickName(userResponse, question);
//TODO: remove nickname ex. "don't call me XXX"
var property = userPropertyService.GetProperty(userResponse, question);
var userData = UserDatabase.UserDatabase.userDatabase.FirstOrDefault(ud => ud != null && ud.userName != null && ud.userName == userResponse.chat.user);
if (userData != null)
{
if (!string.IsNullOrEmpty(nickName))
{
if (!userData.nickNames.Contains(nickName))
{
userData.nickNames.Add(nickName);
}
}
if (!string.IsNullOrEmpty(property.name) && !string.IsNullOrEmpty(property.value) && !string.IsNullOrEmpty(property.source))
{
userData.properties.Add(property);
}
userData.derivedProperties.AddRange(userDerivedPropertyService.GetDerivedProperties(userResponse, property, userData));
userSaveService.SaveUserData(userData);
}
else
{
if (!string.IsNullOrWhiteSpace(userResponse.chat.user))
{
userData = new UserData(userResponse.chat.user);
if (!string.IsNullOrEmpty(nickName))
{
if (!userData.nickNames.Contains(nickName))
{
userData.nickNames.Add(nickName);
}
}
userData.derivedProperties.AddRange(userDerivedPropertyService.GetDerivedProperties(userResponse, property, userData));
UserDatabase.UserDatabase.userDatabase.Add(userData);
userSaveService.SaveUserData(userData);
}
}
var otherUserProperty = otherUserPropertyService.GetOtherUserProperty(userResponse, UserDatabase.UserDatabase.userDatabase);
userData = UserDatabase.UserDatabase.userDatabase.FirstOrDefault(ud => ud != null && ud.userName != null && ud.userName == otherUserProperty.userName);
if (userData == null)
{
userData = UserDatabase.UserDatabase.userDatabase.FirstOrDefault(ud => ud != null && ud.userName != null && ud.nickNames.Contains(otherUserProperty.userName));
}
if (userData != null)
{
if (!string.IsNullOrEmpty(otherUserProperty.userProperty.name) && !string.IsNullOrEmpty(otherUserProperty.userProperty.value) && !string.IsNullOrEmpty(otherUserProperty.userProperty.source))
{
userData.properties.Add(otherUserProperty.userProperty);
}
userSaveService.SaveUserData(userData);
}
}
public ChatResponse GetPropertyResponse(AnalyzedChat analyzedChat, UserData userData)
{
var requestedPropertyName = getRequestedProperty(analyzedChat);
if (!string.IsNullOrEmpty(requestedPropertyName))
{
var requestedProperty = _propertyValueService.getSelfPropertyByValue(requestedPropertyName, userData);
if (!string.IsNullOrEmpty(requestedProperty.value))
{
var confidence = 1.0;
if (requestedProperty.source != analyzedChat.botName)
{
confidence = .75;
}
var response = new List <string>();
response.Add(getYourPropertySentence(requestedProperty));
return(new ChatResponse {
confidence = confidence, response = response
});
}
}
return(new ChatResponse {
confidence = 0, response = new List <string>()
});
}
private ChatResponse GetSubjectDefinition(AnalyzedChat request)
{
if (request.naturalLanguageData.sentences != null && request.naturalLanguageData.sentences[0].Subject.Lemmas != null)
{
var definitions = new List <ChatResponse>();
var definition = GetDefinition(request.naturalLanguageData.sentences[0].Subject.Lemmas);
if (definition.confidence > 0)
{
definitions.Add(definition);
}
if (definitions.Count() > 0)
{
var bestDefinition = definitions.OrderByDescending(x => x.confidence).First();
bestDefinition.confidence = urbanDictionaryTokenScore;
return(bestDefinition);
}
}
return(new ChatResponse()
{
response = new List <string>(), confidence = 0
});
}
public ChatResponse GetYourPropertyResponse(AnalyzedChat analyzedChat, UserData userData)
{
var requestedPropertyName = getRequestedProperty(analyzedChat);
if (!string.IsNullOrEmpty(requestedPropertyName))
{
var requestedProperty = _propertyValueService.getSelfPropertyByValue(requestedPropertyName, userData);
if (!string.IsNullOrEmpty(requestedProperty.value))
{
var confidence = 1.0;
if (requestedProperty.source != userData.userName) //TODO: determine confidence based on source/user relationship, trustworthiness, etc.
{
confidence = .75;
}
var response = new List <string>();
response.Add(getYourPropertySentence(requestedProperty));
return(new ChatResponse {
confidence = confidence, response = response
});
}
}
return(new ChatResponse {
confidence = 0, response = new List <string>()
});
}
private UserNameAndProperty getOtherRequestedPropertyName(AnalyzedChat analyzedChat, ConcurrentBag <UserData> users)
{
foreach (var regex in otherPropertySearch)
{
var match = getOtherPropertyMatch(analyzedChat.chat.message, regex);
if (!string.IsNullOrWhiteSpace(match.name) && !string.IsNullOrWhiteSpace(match.value) && _userNaturalLanguageService.isNaturalLanguageSelfProperty(analyzedChat, match.value))
{
var userMatches = users.Where(u => "@" + u.userName == match.name || u.userName == match.name);
if (userMatches.Count() == 0)
{
userMatches = users.Where(u => u.nickNames.Contains(match.name));
}
if (userMatches.Count() > 0)
{
var userWithPropertyMatches = userMatches.Where(u => u.properties.Any(p => p.name == "self" && p.value == match.value && p.source == u.userName));
if (userWithPropertyMatches.Count() == 0)
{
userWithPropertyMatches = userMatches.Where(u => u.properties.Any(p => p.name == "self" && p.value == match.value));
}
if (userWithPropertyMatches.Count() > 0)
{
var property = userWithPropertyMatches.First().properties.Where(p => p.name == "self" && p.value == match.value).First();
return(new UserNameAndProperty()
{
userName = match.name, userProperty = property
});
}
}
}
}
return(new UserNameAndProperty()
{
userName = string.Empty, userProperty = new UserProperty()
});
}
public void GetReadingLevel()
{
var service = new ConversationReadingLevelService();
var responses = new List <AnalyzedChat>();
var analyzedChat = new AnalyzedChat
{
botName = "sharkbot",
chat = new Chat
{
message = "suh",
user = "******",
botName = "sharkbot"
}
};
responses.Add(analyzedChat);
var result = service.GetReadingLevel(responses);
Assert.Equal(-6.8, result.AutomatedReadabilityIndex);
Assert.Equal(1.6, result.ColemanLiauIndex);
Assert.Equal(-3.4, result.FleschKincaidGradeLevel);
Assert.Equal(121.2, result.FleschKincaidReadingEase);
Assert.Equal(0.4, result.GunningFogScore);
Assert.Equal(1.8, result.SMOGIndex);
}
public void GetSameSubject()
{
var sentences = new List <Sentence>();
sentences.Add(new Sentence {
Subject = new Token {
Lemmas = "ninja"
}
});
sentences.Add(new Sentence {
Subject = new Token {
Lemmas = "ninja"
}
});
var analyzedChat = new AnalyzedChat
{
naturalLanguageData = new NaturalLanguageData {
sentences = sentences
}
};
var service = new ResponseSubjectService();
var result = service.GetSubjects(analyzedChat);
Assert.Single(result);
Assert.Equal("ninja", result[0].Lemmas);
Assert.Equal(2, result[0].OccurenceCount);
}
public void GetConversationSubject()
{
var responses = new List <AnalyzedChat>();
var sentences = new List <Sentence>();
sentences.Add(new Sentence {
Subject = new Token {
Word = "ninjas", Lemmas = "ninja"
}
});
var analyzedChat = new AnalyzedChat
{
naturalLanguageData = new NaturalLanguageData {
sentences = sentences
}
};
responses.Add(analyzedChat);
analyzedChat = new AnalyzedChat
{
naturalLanguageData = new NaturalLanguageData {
sentences = sentences
}
};
responses.Add(analyzedChat);
var service = new ConversationSubjectService(new ResponseSubjectService());
var result = service.GetConversationSubjects(responses);
Assert.Single(result);
Assert.Equal(2, result[0].OccurenceCount);
}
public void AnalyzeEmojisConversationAsync()
{
var service = GetAnalyzationService();
var conversation = new Conversation();
conversation.name = "Test";
conversation.responses = new List <AnalyzedChat>();
var analyzedChat = new AnalyzedChat
{
botName = "sharkbot",
chat = new Chat
{
message = "I'm a 37.5% 🤔 ",
user = "******",
botName = "sharkbot"
}
};
conversation.responses.Add(analyzedChat);
var result = service.AnalyzeConversationAsync(conversation);
Assert.Equal(conversation.name, result.name);
Assert.False(result.groupChat);
Assert.Single(result.responses);
}
public void AnalyzeWeirdCharactersConversationAsync()
{
var service = GetAnalyzationService();
var conversation = new Conversation();
conversation.name = "Test";
conversation.responses = new List <AnalyzedChat>();
var analyzedChat = new AnalyzedChat
{
botName = "sharkbot",
chat = new Chat
{
message = "⣿⠄⡇⢸⣟⠄⠁⢸⡽⠖⠛⠈⡉⣉⠉⠋⣁⢘⠉⢉⠛⡿⢿⣿⣿⣿⣿⣿⣿⣿ ⣷⣶⣷⣤⠄⣠⠖⠁⠄⠂⠁⠄⠄⠉⠄⠄⠎⠄⠠⠎⢐⠄⢑⣛⠻⣿⣿⣿⣿⣿ ⣿⣿⣿⠓⠨⠄⠄⠄⠄⠄⠄⠄⠄⠄⠄⠄⠈⠐⠅⠄⠉⠄⠗⠆⣸⣿⣿⣿⣿⣿ ⣿⣿⣿⡣⠁⠄⠄⠄⠄⠄⠄⠄⠄⠄⢰⣤⣦⠄⠄⠄⠄⠄⠄⠄⡀⡙⣿⣿⣿⣿ ⣿⣿⡛⠄⠄⠄⠄⠄⠄⠄⠄⠄⠄⠔⠿⡿⠿⠒⠄⠠⢤⡀⡀⠄⠁⠄⢻⣿⣿⣿ ⣿⣿⠄⠄⠄⠄⠄⠄⣠⡖⠄⠁⠁⠄⠄⠄⠄⠄⠄⠄⣽⠟⡖⠄⠄⠄⣼⣿⣿⣿ ⣿⣿⠄⠄⠄⠄⠄⠄⢠⣠⣀⠄⠄⠄⠄⢀⣾⣧⠄⠂⠸⣈⡏⠄⠄⠄⣿⣿⣿⣿ ⣿⣿⡞⠄⠄⠄⠄⠄⢸⣿⣶⣶⣶⣶⣶⡿⢻⡿⣻⣶⣿⣿⡇⠄⠄⠄⣿⣿⣿⣿ ⣿⣿⡷⡂⠄⠄⠁⠄⠸⣿⣿⣿⣿⣿⠟⠛⠉⠉⠙⠛⢿⣿⡇⠄⠄⢀⣿⣿⣿⣿ ⣶⣶⠃⠄⠄⠄⠄⠄⠄⣾⣿⣿⡿⠁⣀⣀⣤⣤⣤⣄⢈⣿⡇⠄⠄⢸⣿⣿⣿⣿ ⣿⣯⠄⠄⠄⠄⠄⠄⠄⢻⣿⣿⣷⣶⣿⣿⣥⣬⣿⣿⣟⣿⠃⠄⠨⠺⢿⣿⣿⣿ ⠱⠂⠄⠄⠄⠄⠄⠄⠄⣬⣸⡝⠿⢿⣿⡿⣿⠻⠟⠻⢫⡁⠄⠄⠄⡐⣾⣿⣿⣿ ⡜⠄⠄⠄⠄⠄⠆⡐⡇⢿⣽⣻⣷⣦⣧⡀⡀⠄⠄⣴⣺⡇⠄⠁⠄⢣⣿⣿⣿⣿ ⠡⠱⠄⠄⠡⠄⢠⣷⠆⢸⣿⣿⣿⣿⣿⣿⣷⣿⣾⣿⣿⡇⠄⠄⠠⠁⠿⣿⣿⣿ ⢀⣲⣧⣷⣿⢂⣄⡉⠄⠘⠿⣿⣿⣿⡟⣻⣯⠿⠟⠋⠉⢰⢦⠄⠊⢾⣷⣮⣽⣛",
user = "******",
botName = "sharkbot"
}
};
conversation.responses.Add(analyzedChat);
var result = service.AnalyzeConversationAsync(conversation);
Assert.Equal(conversation.name, result.name);
Assert.False(result.groupChat);
Assert.Single(result.responses);
}
public void AnalyzeSpecialCharactersConversationAsync()
{
var service = GetAnalyzationService();
var conversation = new Conversation();
conversation.name = "Test";
conversation.responses = new List <AnalyzedChat>();
var analyzedChat = new AnalyzedChat
{
botName = "sharkbot",
chat = new Chat
{
message = "尸尺乇尸卂尺乇 下口尺 丅尺口凵乃乚乇 卂𠘨刀 从卂长乇 工丅 刀口凵乃乚乇!",
user = "******",
botName = "sharkbot"
}
};
conversation.responses.Add(analyzedChat);
var result = service.AnalyzeConversationAsync(conversation);
Assert.Equal(conversation.name, result.name);
Assert.False(result.groupChat);
Assert.Single(result.responses);
}
private ChatResponse GetDefinitionFromQuestion(AnalyzedChat chat, AnalyzedChat previousChat)
{
var searchText = GetSearchText(chat);
foreach (var regex in definitionSearches)
{
var word = getMatch(searchText, regex);
if (word != string.Empty)
{
if (excludedWords.Any(e => word.Contains(e)))
{
var tripletDefinition = GetSubjectDefinition(previousChat);
if (tripletDefinition.confidence > 0)
{
return(tripletDefinition);
}
}
}
}
return(new ChatResponse()
{
response = new List <string>(), confidence = 0
});
}
public List <ConversationSubject> GetSubjects(AnalyzedChat response)
{
var subjects = new List <ConversationSubject>();
foreach (var sentence in response.naturalLanguageData.sentences)
{
foreach (var token in sentence.tokens)
{
if (token.POSTag == "NN" || token.POSTag == "NNP" || token.POSTag == "NNS" || token.POSTag == "NNPS")
{
var index = subjects.FindIndex(s => s.subjectWords.Contains(token.Stem));
if (index >= 0)
{
subjects[index].occurenceCount++;
}
else
{
var subject = new ConversationSubject();
subject.occurenceCount = 1;
subject.subjectWords = new List <string>();
subject.subjectWords.Add(token.Stem);
subjects.Add(subject);
}
}
}
}
return(subjects);
}
private MatchChat GetMatch(Conversation targetConversation, AnalyzedChat existingResponse, double subjectMatchConfidence, double readingLevelMatchConfidence, bool existingGroupChat, string userlessReply, List <string> pathSubjects)
{
//TODO: add user comparison and user similarity to the algorithm for confidence, if user has same property as bot, etc.
var targetResponse = targetConversation.responses.Last();
var matchChat = new MatchChat
{
matchConfidence = 0,
analyzedChat = existingResponse
};
if (existingResponse.naturalLanguageData.responseConfidence == 0)
{
return(matchChat);
}
var uniqueConfidence = _uniqueConfidenceService.GetUniqueConfidence(userlessReply, targetConversation);
var replyConfidence = existingResponse.naturalLanguageData.responseConfidence;
var confidenceMultiplier = uniqueConfidence * replyConfidence;
var replyMatchConfidence = _matchConfidenceService.GetMatchConfidence(targetResponse.naturalLanguageData, existingResponse.naturalLanguageData, targetResponse.botName);
var replyPartScore = replyMatchConfidence * replyMatchRatio;
//TODO: lower the value of replyMatchConfidence if it's a shorter reply like "yes"
var conversationProximityScore = _subjectConfidenceService.GetConversationProximityMatchConfidence(targetResponse.naturalLanguageData.proximitySubjects, existingResponse.naturalLanguageData.proximitySubjects);
var proximityPartScore = conversationProximityScore * conversationProximityRatio;
var subjectMatchPartScore = subjectMatchConfidence * subjectMatchRatio;
var readingLevelMatchPartScore = readingLevelMatchConfidence * readingLevelMatchRatio;
var groupChatMatchConfidence = _groupChatConfidenceService.GetGroupChatConfidence(targetConversation.groupChat, existingGroupChat);
var groupChatPartScore = groupChatMatchConfidence * groupChatRatio;
var confidence = (replyPartScore + proximityPartScore + subjectMatchPartScore + readingLevelMatchPartScore + groupChatPartScore) * confidenceMultiplier;
var length = 0;
if (targetResponse.naturalLanguageData.sentences != null)
{
length = targetResponse.naturalLanguageData.sentences.SelectMany(s => s.Tokens).Count();
}
if (length < 5)
{
var exponent = 6 - length;
confidence = Math.Pow(confidence, exponent);
}
var bonusConfidence = getBonusConfidence(existingResponse, pathSubjects);
confidence = confidence + bonusConfidence;
matchChat.matchConfidence = confidence;
return(matchChat);
}
public List <UserProperty> GetDerivedProperties(AnalyzedChat analyzedChat, UserProperty givenProperty, UserData userData)
{
var properties = new List <UserProperty>();
properties.AddRange(getSex(analyzedChat, givenProperty, userData));
//TODO: add more of these, address, age (note time it was retrieved and factor that into response), derive from asl, etc.
//TODO: use machine learning to determine sex and other properties, train on conversations with known males and females or known specific property versus know doesn't have property
return(properties);
}
private string GetSearchText(AnalyzedChat chat)
{
var searchText = chat.chat.message;
if (searchText.Contains(chat.botName))
{
searchText = searchText.Replace("@" + chat.botName, string.Empty).Replace(chat.botName, string.Empty);
}
return(searchText);
}
public UserProperty GetProperty(AnalyzedChat analyzedChat, AnalyzedChat question)
{
var property = getPropertyFromResponse(analyzedChat, question);
if (string.IsNullOrEmpty(property.name))
{
property = _propertyFromQuestionService.getPropertyFromQuestion(analyzedChat, question);
}
return(property);
}
private double getBonusConfidence(AnalyzedChat existingResponse, List <string> pathSubjects)
{
foreach (var conversationSubjects in existingResponse.naturalLanguageData.subjects)
{
if (pathSubjects.Contains(conversationSubjects.Lemmas))
{
return(goalBonus);
}
}
return(0);
}
private string GetDefinitionFormatMatch(AnalyzedChat chat)
{
var searchText = GetSearchText(chat);
if (excludedWords.Any(e => searchText.Contains(e)))
{
return(string.Empty);
}
return(GetDefinitionMatch(searchText));
}
//TODO: determine if someone else is calling someone by a nickname. If there is a proper noun and it is similar to their username, for example username sharknice and being called sharkie 4 consecutive letters match
public string GetNickName(AnalyzedChat analyzedChat, AnalyzedChat question)
{
var nickName = getNickNameFromResponse(analyzedChat);
if (nickName == string.Empty)
{
nickName = getNickNameFromQuestion(analyzedChat, question);
}
return(nickName);
}
private string getRequestedProperty(AnalyzedChat analyzedChat)
{
foreach (var regex in selfPropertySearch)
{
var match = getPropertyMatch(analyzedChat.chat.message, regex);
if (!string.IsNullOrWhiteSpace(match) && _userNaturalLanguageService.isNaturalLanguageSelfProperty(analyzedChat, match))
{
return(match);
}
}
return(string.Empty);
}
private List <AnalyzedChat> getAnalyzedChatsFromReactions(List <Reaction> reactions)
{
var analyzedChats = new List <AnalyzedChat>();
foreach (var reaction in reactions)
{
var analyzedChat = new AnalyzedChat();
analyzedChat.chat = new Chat();
analyzedChat.chat.message = reaction.reaction;
analyzedChats.Add(analyzedChat);
}
return(analyzedChats);
}
private bool isNaturalLanguageName(AnalyzedChat chat, string match)
{
foreach (var sentence in chat.naturalLanguageData.sentences)
{
foreach (var token in sentence.Tokens)
{
if (token.Word == match) //TODO: use ner tag to see if person
{
return(token.PosTag == "NN" || token.PosTag == "NNP" || token.PosTag == "VBG");
}
}
}
return(false);
}
public bool isNaturalLanguagePropertyValue(AnalyzedChat chat, string match)
{
foreach (var sentence in chat.naturalLanguageData.sentences)
{
foreach (var token in sentence.Tokens)
{
if (token.Word == match)
{
return(token.PosTag == "JJ");
}
}
}
return(false);
}
public bool isNaturalLanguagePropertyName(AnalyzedChat chat, string match)
{
foreach (var sentence in chat.naturalLanguageData.sentences)
{
foreach (var token in sentence.tokens)
{
if (token.Lexeme == match)
{
return(token.POSTag == "NN" || token.POSTag == "NNP" || token.POSTag == "NNS");
}
}
}
return(false);
}
