public List <ConversationSubject> GetSubjects(AnalyzedChat response) { var subjects = new List <ConversationSubject>(); if (response.naturalLanguageData.sentences == null) { return(subjects); } foreach (var sentence in response.naturalLanguageData.sentences) { var token = sentence.Subject; if (token != null && !string.IsNullOrWhiteSpace(token.Lemmas)) { var index = subjects.FindIndex(s => s.Lemmas == token.Lemmas); if (index >= 0) { subjects[index].OccurenceCount++; } else { var subject = new ConversationSubject { OccurenceCount = 1, Lemmas = token.Lemmas }; subjects.Add(subject); } } } return(subjects); }
public void GetMultipleSubjects() { var sentences = new List <Sentence>(); sentences.Add(new Sentence { Subject = new Token { Lemmas = "ninja" } }); sentences.Add(new Sentence { Subject = new Token { Lemmas = "people" } }); var analyzedChat = new AnalyzedChat { naturalLanguageData = new NaturalLanguageData { sentences = sentences } }; var service = new ResponseSubjectService(); var result = service.GetSubjects(analyzedChat); Assert.Equal(2, result.Count); }
private ChatResponse GetTripletDefinition(AnalyzedChat request) { if (request.naturalLanguageData.sentences[0].triplets.objectTriplet != null) { var definitions = new List <ChatResponse>(); foreach (var targetToken in request.naturalLanguageData.sentences[0].triplets.objectTriplet.chunk.tokens) { if (targetToken.POSTag == "NN" || targetToken.POSTag == "NNS" || targetToken.POSTag == "NNP" || targetToken.POSTag == "NNPS" || targetToken.POSTag == "VBG") { var definition = GetDefinition(targetToken.Lexeme); if (definition.confidence > 0) { definitions.Add(definition); } } } if (definitions.Count() > 0) { var bestDefinition = definitions.OrderByDescending(x => x.confidence).First(); bestDefinition.confidence = urbanDictionaryTokenScore; return(bestDefinition); } } return(new ChatResponse() { response = new List <string>(), confidence = 0 }); }
public void GetProximitySubjects() { var responses = new List <AnalyzedChat>(); var sentences = new List <Sentence>(); sentences.Add(new Sentence { Subject = new Token { Word = "ninjas", Lemmas = "ninja" } }); var analyzedChat = new AnalyzedChat { naturalLanguageData = new NaturalLanguageData { sentences = sentences } }; responses.Add(analyzedChat); analyzedChat = new AnalyzedChat { naturalLanguageData = new NaturalLanguageData { sentences = sentences } }; responses.Add(analyzedChat); var conversation = new Conversation(); conversation.responses = responses; var service = new ConversationSubjectService(new ResponseSubjectService()); var result = service.GetProximitySubjects(conversation, 0); Assert.Single(result); }
public bool UpdateDatabases(ConversationRequest conversationRequest) { if (_conversationService.ConversationLength(conversationRequest.name, conversationRequest.type) >= conversationRequest.responses.Count()) { return(false); } var conversation = new Conversation { name = conversationRequest.name, responses = conversationRequest.responses }; var analyzedConversation = _analyzationService.AnalyzeConversationAsync(conversation); var conversationUdpdated = _covnersationUpdateService.UpdateConversation(analyzedConversation, conversationRequest.type); AnalyzedChat previousChat = null; foreach (var analyziedChat in analyzedConversation.responses) { _userService.UpdateUsers(analyziedChat, previousChat); previousChat = analyziedChat; } return(conversationUdpdated); }
public void UpdateUsers(AnalyzedChat userResponse, AnalyzedChat question) { var nickName = userNickNameService.GetNickName(userResponse, question); //TODO: remove nickname ex. "don't call me XXX" var property = userPropertyService.GetProperty(userResponse, question); var userData = UserDatabase.UserDatabase.userDatabase.FirstOrDefault(ud => ud != null && ud.userName != null && ud.userName == userResponse.chat.user); if (userData != null) { if (!string.IsNullOrEmpty(nickName)) { if (!userData.nickNames.Contains(nickName)) { userData.nickNames.Add(nickName); } } if (!string.IsNullOrEmpty(property.name) && !string.IsNullOrEmpty(property.value) && !string.IsNullOrEmpty(property.source)) { userData.properties.Add(property); } userData.derivedProperties.AddRange(userDerivedPropertyService.GetDerivedProperties(userResponse, property, userData)); userSaveService.SaveUserData(userData); } else { if (!string.IsNullOrWhiteSpace(userResponse.chat.user)) { userData = new UserData(userResponse.chat.user); if (!string.IsNullOrEmpty(nickName)) { if (!userData.nickNames.Contains(nickName)) { userData.nickNames.Add(nickName); } } userData.derivedProperties.AddRange(userDerivedPropertyService.GetDerivedProperties(userResponse, property, userData)); UserDatabase.UserDatabase.userDatabase.Add(userData); userSaveService.SaveUserData(userData); } } var otherUserProperty = otherUserPropertyService.GetOtherUserProperty(userResponse, UserDatabase.UserDatabase.userDatabase); userData = UserDatabase.UserDatabase.userDatabase.FirstOrDefault(ud => ud != null && ud.userName != null && ud.userName == otherUserProperty.userName); if (userData == null) { userData = UserDatabase.UserDatabase.userDatabase.FirstOrDefault(ud => ud != null && ud.userName != null && ud.nickNames.Contains(otherUserProperty.userName)); } if (userData != null) { if (!string.IsNullOrEmpty(otherUserProperty.userProperty.name) && !string.IsNullOrEmpty(otherUserProperty.userProperty.value) && !string.IsNullOrEmpty(otherUserProperty.userProperty.source)) { userData.properties.Add(otherUserProperty.userProperty); } userSaveService.SaveUserData(userData); } }
public ChatResponse GetPropertyResponse(AnalyzedChat analyzedChat, UserData userData) { var requestedPropertyName = getRequestedProperty(analyzedChat); if (!string.IsNullOrEmpty(requestedPropertyName)) { var requestedProperty = _propertyValueService.getSelfPropertyByValue(requestedPropertyName, userData); if (!string.IsNullOrEmpty(requestedProperty.value)) { var confidence = 1.0; if (requestedProperty.source != analyzedChat.botName) { confidence = .75; } var response = new List <string>(); response.Add(getYourPropertySentence(requestedProperty)); return(new ChatResponse { confidence = confidence, response = response }); } } return(new ChatResponse { confidence = 0, response = new List <string>() }); }
private ChatResponse GetSubjectDefinition(AnalyzedChat request) { if (request.naturalLanguageData.sentences != null && request.naturalLanguageData.sentences[0].Subject.Lemmas != null) { var definitions = new List <ChatResponse>(); var definition = GetDefinition(request.naturalLanguageData.sentences[0].Subject.Lemmas); if (definition.confidence > 0) { definitions.Add(definition); } if (definitions.Count() > 0) { var bestDefinition = definitions.OrderByDescending(x => x.confidence).First(); bestDefinition.confidence = urbanDictionaryTokenScore; return(bestDefinition); } } return(new ChatResponse() { response = new List <string>(), confidence = 0 }); }
public ChatResponse GetYourPropertyResponse(AnalyzedChat analyzedChat, UserData userData) { var requestedPropertyName = getRequestedProperty(analyzedChat); if (!string.IsNullOrEmpty(requestedPropertyName)) { var requestedProperty = _propertyValueService.getSelfPropertyByValue(requestedPropertyName, userData); if (!string.IsNullOrEmpty(requestedProperty.value)) { var confidence = 1.0; if (requestedProperty.source != userData.userName) //TODO: determine confidence based on source/user relationship, trustworthiness, etc. { confidence = .75; } var response = new List <string>(); response.Add(getYourPropertySentence(requestedProperty)); return(new ChatResponse { confidence = confidence, response = response }); } } return(new ChatResponse { confidence = 0, response = new List <string>() }); }
private UserNameAndProperty getOtherRequestedPropertyName(AnalyzedChat analyzedChat, ConcurrentBag <UserData> users) { foreach (var regex in otherPropertySearch) { var match = getOtherPropertyMatch(analyzedChat.chat.message, regex); if (!string.IsNullOrWhiteSpace(match.name) && !string.IsNullOrWhiteSpace(match.value) && _userNaturalLanguageService.isNaturalLanguageSelfProperty(analyzedChat, match.value)) { var userMatches = users.Where(u => "@" + u.userName == match.name || u.userName == match.name); if (userMatches.Count() == 0) { userMatches = users.Where(u => u.nickNames.Contains(match.name)); } if (userMatches.Count() > 0) { var userWithPropertyMatches = userMatches.Where(u => u.properties.Any(p => p.name == "self" && p.value == match.value && p.source == u.userName)); if (userWithPropertyMatches.Count() == 0) { userWithPropertyMatches = userMatches.Where(u => u.properties.Any(p => p.name == "self" && p.value == match.value)); } if (userWithPropertyMatches.Count() > 0) { var property = userWithPropertyMatches.First().properties.Where(p => p.name == "self" && p.value == match.value).First(); return(new UserNameAndProperty() { userName = match.name, userProperty = property }); } } } } return(new UserNameAndProperty() { userName = string.Empty, userProperty = new UserProperty() }); }
public void GetReadingLevel() { var service = new ConversationReadingLevelService(); var responses = new List <AnalyzedChat>(); var analyzedChat = new AnalyzedChat { botName = "sharkbot", chat = new Chat { message = "suh", user = "******", botName = "sharkbot" } }; responses.Add(analyzedChat); var result = service.GetReadingLevel(responses); Assert.Equal(-6.8, result.AutomatedReadabilityIndex); Assert.Equal(1.6, result.ColemanLiauIndex); Assert.Equal(-3.4, result.FleschKincaidGradeLevel); Assert.Equal(121.2, result.FleschKincaidReadingEase); Assert.Equal(0.4, result.GunningFogScore); Assert.Equal(1.8, result.SMOGIndex); }
public void GetSameSubject() { var sentences = new List <Sentence>(); sentences.Add(new Sentence { Subject = new Token { Lemmas = "ninja" } }); sentences.Add(new Sentence { Subject = new Token { Lemmas = "ninja" } }); var analyzedChat = new AnalyzedChat { naturalLanguageData = new NaturalLanguageData { sentences = sentences } }; var service = new ResponseSubjectService(); var result = service.GetSubjects(analyzedChat); Assert.Single(result); Assert.Equal("ninja", result[0].Lemmas); Assert.Equal(2, result[0].OccurenceCount); }
public void GetConversationSubject() { var responses = new List <AnalyzedChat>(); var sentences = new List <Sentence>(); sentences.Add(new Sentence { Subject = new Token { Word = "ninjas", Lemmas = "ninja" } }); var analyzedChat = new AnalyzedChat { naturalLanguageData = new NaturalLanguageData { sentences = sentences } }; responses.Add(analyzedChat); analyzedChat = new AnalyzedChat { naturalLanguageData = new NaturalLanguageData { sentences = sentences } }; responses.Add(analyzedChat); var service = new ConversationSubjectService(new ResponseSubjectService()); var result = service.GetConversationSubjects(responses); Assert.Single(result); Assert.Equal(2, result[0].OccurenceCount); }
public void AnalyzeEmojisConversationAsync() { var service = GetAnalyzationService(); var conversation = new Conversation(); conversation.name = "Test"; conversation.responses = new List <AnalyzedChat>(); var analyzedChat = new AnalyzedChat { botName = "sharkbot", chat = new Chat { message = "I'm a 37.5% 🤔 ", user = "******", botName = "sharkbot" } }; conversation.responses.Add(analyzedChat); var result = service.AnalyzeConversationAsync(conversation); Assert.Equal(conversation.name, result.name); Assert.False(result.groupChat); Assert.Single(result.responses); }
public void AnalyzeWeirdCharactersConversationAsync() { var service = GetAnalyzationService(); var conversation = new Conversation(); conversation.name = "Test"; conversation.responses = new List <AnalyzedChat>(); var analyzedChat = new AnalyzedChat { botName = "sharkbot", chat = new Chat { message = "⣿⠄⡇⢸⣟⠄⠁⢸⡽⠖⠛⠈⡉⣉⠉⠋⣁⢘⠉⢉⠛⡿⢿⣿⣿⣿⣿⣿⣿⣿ ⣷⣶⣷⣤⠄⣠⠖⠁⠄⠂⠁⠄⠄⠉⠄⠄⠎⠄⠠⠎⢐⠄⢑⣛⠻⣿⣿⣿⣿⣿ ⣿⣿⣿⠓⠨⠄⠄⠄⠄⠄⠄⠄⠄⠄⠄⠄⠈⠐⠅⠄⠉⠄⠗⠆⣸⣿⣿⣿⣿⣿ ⣿⣿⣿⡣⠁⠄⠄⠄⠄⠄⠄⠄⠄⠄⢰⣤⣦⠄⠄⠄⠄⠄⠄⠄⡀⡙⣿⣿⣿⣿ ⣿⣿⡛⠄⠄⠄⠄⠄⠄⠄⠄⠄⠄⠔⠿⡿⠿⠒⠄⠠⢤⡀⡀⠄⠁⠄⢻⣿⣿⣿ ⣿⣿⠄⠄⠄⠄⠄⠄⣠⡖⠄⠁⠁⠄⠄⠄⠄⠄⠄⠄⣽⠟⡖⠄⠄⠄⣼⣿⣿⣿ ⣿⣿⠄⠄⠄⠄⠄⠄⢠⣠⣀⠄⠄⠄⠄⢀⣾⣧⠄⠂⠸⣈⡏⠄⠄⠄⣿⣿⣿⣿ ⣿⣿⡞⠄⠄⠄⠄⠄⢸⣿⣶⣶⣶⣶⣶⡿⢻⡿⣻⣶⣿⣿⡇⠄⠄⠄⣿⣿⣿⣿ ⣿⣿⡷⡂⠄⠄⠁⠄⠸⣿⣿⣿⣿⣿⠟⠛⠉⠉⠙⠛⢿⣿⡇⠄⠄⢀⣿⣿⣿⣿ ⣶⣶⠃⠄⠄⠄⠄⠄⠄⣾⣿⣿⡿⠁⣀⣀⣤⣤⣤⣄⢈⣿⡇⠄⠄⢸⣿⣿⣿⣿ ⣿⣯⠄⠄⠄⠄⠄⠄⠄⢻⣿⣿⣷⣶⣿⣿⣥⣬⣿⣿⣟⣿⠃⠄⠨⠺⢿⣿⣿⣿ ⠱⠂⠄⠄⠄⠄⠄⠄⠄⣬⣸⡝⠿⢿⣿⡿⣿⠻⠟⠻⢫⡁⠄⠄⠄⡐⣾⣿⣿⣿ ⡜⠄⠄⠄⠄⠄⠆⡐⡇⢿⣽⣻⣷⣦⣧⡀⡀⠄⠄⣴⣺⡇⠄⠁⠄⢣⣿⣿⣿⣿ ⠡⠱⠄⠄⠡⠄⢠⣷⠆⢸⣿⣿⣿⣿⣿⣿⣷⣿⣾⣿⣿⡇⠄⠄⠠⠁⠿⣿⣿⣿ ⢀⣲⣧⣷⣿⢂⣄⡉⠄⠘⠿⣿⣿⣿⡟⣻⣯⠿⠟⠋⠉⢰⢦⠄⠊⢾⣷⣮⣽⣛", user = "******", botName = "sharkbot" } }; conversation.responses.Add(analyzedChat); var result = service.AnalyzeConversationAsync(conversation); Assert.Equal(conversation.name, result.name); Assert.False(result.groupChat); Assert.Single(result.responses); }
public void AnalyzeSpecialCharactersConversationAsync() { var service = GetAnalyzationService(); var conversation = new Conversation(); conversation.name = "Test"; conversation.responses = new List <AnalyzedChat>(); var analyzedChat = new AnalyzedChat { botName = "sharkbot", chat = new Chat { message = "尸尺乇尸卂尺乇 下口尺 丅尺口凵乃乚乇 卂𠘨刀 从卂长乇 工丅 刀口凵乃乚乇!", user = "******", botName = "sharkbot" } }; conversation.responses.Add(analyzedChat); var result = service.AnalyzeConversationAsync(conversation); Assert.Equal(conversation.name, result.name); Assert.False(result.groupChat); Assert.Single(result.responses); }
private ChatResponse GetDefinitionFromQuestion(AnalyzedChat chat, AnalyzedChat previousChat) { var searchText = GetSearchText(chat); foreach (var regex in definitionSearches) { var word = getMatch(searchText, regex); if (word != string.Empty) { if (excludedWords.Any(e => word.Contains(e))) { var tripletDefinition = GetSubjectDefinition(previousChat); if (tripletDefinition.confidence > 0) { return(tripletDefinition); } } } } return(new ChatResponse() { response = new List <string>(), confidence = 0 }); }
public List <ConversationSubject> GetSubjects(AnalyzedChat response) { var subjects = new List <ConversationSubject>(); foreach (var sentence in response.naturalLanguageData.sentences) { foreach (var token in sentence.tokens) { if (token.POSTag == "NN" || token.POSTag == "NNP" || token.POSTag == "NNS" || token.POSTag == "NNPS") { var index = subjects.FindIndex(s => s.subjectWords.Contains(token.Stem)); if (index >= 0) { subjects[index].occurenceCount++; } else { var subject = new ConversationSubject(); subject.occurenceCount = 1; subject.subjectWords = new List <string>(); subject.subjectWords.Add(token.Stem); subjects.Add(subject); } } } } return(subjects); }
private MatchChat GetMatch(Conversation targetConversation, AnalyzedChat existingResponse, double subjectMatchConfidence, double readingLevelMatchConfidence, bool existingGroupChat, string userlessReply, List <string> pathSubjects) { //TODO: add user comparison and user similarity to the algorithm for confidence, if user has same property as bot, etc. var targetResponse = targetConversation.responses.Last(); var matchChat = new MatchChat { matchConfidence = 0, analyzedChat = existingResponse }; if (existingResponse.naturalLanguageData.responseConfidence == 0) { return(matchChat); } var uniqueConfidence = _uniqueConfidenceService.GetUniqueConfidence(userlessReply, targetConversation); var replyConfidence = existingResponse.naturalLanguageData.responseConfidence; var confidenceMultiplier = uniqueConfidence * replyConfidence; var replyMatchConfidence = _matchConfidenceService.GetMatchConfidence(targetResponse.naturalLanguageData, existingResponse.naturalLanguageData, targetResponse.botName); var replyPartScore = replyMatchConfidence * replyMatchRatio; //TODO: lower the value of replyMatchConfidence if it's a shorter reply like "yes" var conversationProximityScore = _subjectConfidenceService.GetConversationProximityMatchConfidence(targetResponse.naturalLanguageData.proximitySubjects, existingResponse.naturalLanguageData.proximitySubjects); var proximityPartScore = conversationProximityScore * conversationProximityRatio; var subjectMatchPartScore = subjectMatchConfidence * subjectMatchRatio; var readingLevelMatchPartScore = readingLevelMatchConfidence * readingLevelMatchRatio; var groupChatMatchConfidence = _groupChatConfidenceService.GetGroupChatConfidence(targetConversation.groupChat, existingGroupChat); var groupChatPartScore = groupChatMatchConfidence * groupChatRatio; var confidence = (replyPartScore + proximityPartScore + subjectMatchPartScore + readingLevelMatchPartScore + groupChatPartScore) * confidenceMultiplier; var length = 0; if (targetResponse.naturalLanguageData.sentences != null) { length = targetResponse.naturalLanguageData.sentences.SelectMany(s => s.Tokens).Count(); } if (length < 5) { var exponent = 6 - length; confidence = Math.Pow(confidence, exponent); } var bonusConfidence = getBonusConfidence(existingResponse, pathSubjects); confidence = confidence + bonusConfidence; matchChat.matchConfidence = confidence; return(matchChat); }
public List <UserProperty> GetDerivedProperties(AnalyzedChat analyzedChat, UserProperty givenProperty, UserData userData) { var properties = new List <UserProperty>(); properties.AddRange(getSex(analyzedChat, givenProperty, userData)); //TODO: add more of these, address, age (note time it was retrieved and factor that into response), derive from asl, etc. //TODO: use machine learning to determine sex and other properties, train on conversations with known males and females or known specific property versus know doesn't have property return(properties); }
private string GetSearchText(AnalyzedChat chat) { var searchText = chat.chat.message; if (searchText.Contains(chat.botName)) { searchText = searchText.Replace("@" + chat.botName, string.Empty).Replace(chat.botName, string.Empty); } return(searchText); }
public UserProperty GetProperty(AnalyzedChat analyzedChat, AnalyzedChat question) { var property = getPropertyFromResponse(analyzedChat, question); if (string.IsNullOrEmpty(property.name)) { property = _propertyFromQuestionService.getPropertyFromQuestion(analyzedChat, question); } return(property); }
private double getBonusConfidence(AnalyzedChat existingResponse, List <string> pathSubjects) { foreach (var conversationSubjects in existingResponse.naturalLanguageData.subjects) { if (pathSubjects.Contains(conversationSubjects.Lemmas)) { return(goalBonus); } } return(0); }
private string GetDefinitionFormatMatch(AnalyzedChat chat) { var searchText = GetSearchText(chat); if (excludedWords.Any(e => searchText.Contains(e))) { return(string.Empty); } return(GetDefinitionMatch(searchText)); }
//TODO: determine if someone else is calling someone by a nickname. If there is a proper noun and it is similar to their username, for example username sharknice and being called sharkie 4 consecutive letters match public string GetNickName(AnalyzedChat analyzedChat, AnalyzedChat question) { var nickName = getNickNameFromResponse(analyzedChat); if (nickName == string.Empty) { nickName = getNickNameFromQuestion(analyzedChat, question); } return(nickName); }
private string getRequestedProperty(AnalyzedChat analyzedChat) { foreach (var regex in selfPropertySearch) { var match = getPropertyMatch(analyzedChat.chat.message, regex); if (!string.IsNullOrWhiteSpace(match) && _userNaturalLanguageService.isNaturalLanguageSelfProperty(analyzedChat, match)) { return(match); } } return(string.Empty); }
private List <AnalyzedChat> getAnalyzedChatsFromReactions(List <Reaction> reactions) { var analyzedChats = new List <AnalyzedChat>(); foreach (var reaction in reactions) { var analyzedChat = new AnalyzedChat(); analyzedChat.chat = new Chat(); analyzedChat.chat.message = reaction.reaction; analyzedChats.Add(analyzedChat); } return(analyzedChats); }
private bool isNaturalLanguageName(AnalyzedChat chat, string match) { foreach (var sentence in chat.naturalLanguageData.sentences) { foreach (var token in sentence.Tokens) { if (token.Word == match) //TODO: use ner tag to see if person { return(token.PosTag == "NN" || token.PosTag == "NNP" || token.PosTag == "VBG"); } } } return(false); }
public bool isNaturalLanguagePropertyValue(AnalyzedChat chat, string match) { foreach (var sentence in chat.naturalLanguageData.sentences) { foreach (var token in sentence.Tokens) { if (token.Word == match) { return(token.PosTag == "JJ"); } } } return(false); }
public bool isNaturalLanguagePropertyName(AnalyzedChat chat, string match) { foreach (var sentence in chat.naturalLanguageData.sentences) { foreach (var token in sentence.tokens) { if (token.Lexeme == match) { return(token.POSTag == "NN" || token.POSTag == "NNP" || token.POSTag == "NNS"); } } } return(false); }