internal static IEnumerable <FeatureCover> GetFeatureCovers(ParsedUtterance expression, ComposedGraph graph)
{
var features = createFeatures(expression, graph);
var index = new FeatureIndex(features);
return(generateCovers(index, 0, expression));
}
private string getAnswer(ParsedUtterance question)
{
var answerHypothesis = InputState.QA.GetBestHypothesis(question);
if (answerHypothesis == null)
{
//we don't know anything about the question
return(null);
}
if (answerHypothesis.Control.Score < AnswerConfidenceThreshold)
{
//sentences are not similar enough
return(null);
}
var answerNodes = InputState.QA.GetAnswer(answerHypothesis);
if (answerNodes == null)
{
//cannot find the node
return(null);
}
var joinedAnswer = string.Join(" and ", answerNodes.Select(a => a.Data));
return(string.Format("It is {0}.", joinedAnswer));
}
private ResponseBase edgeHintInput(ParsedUtterance utterance, DialogActBase act)
{
if (act.IsDontKnow)
{
return(new NotUsefulContinuationAct(startNewFrame()));
}
var edgeHint = utterance.OriginalSentence;
var intextFormat = getIntextFormat(edgeHint);
if (act.IsNegate && intextFormat == null)
{
return(new RequestWordingAct());
}
if (act.IsAffirm)
{
//force users to really output something
if (_initialEdgeHypothesis != _currentEdgeHypothesis)
{
HadInformativeInput = true;
CanBeCompleted = true;
_currentEdgeTopic.AddExpression(_currentEdgeHypothesis);
return(new UsefulContinuationAct(startNewFrame()));
}
else
{
return(new NotUsefulContinuationAct(startNewFrame()));
}
}
if (intextFormat == null)
{
var taggedInput = tagEdgeHint(edgeHint);
if (getWords(taggedInput).Length < 2)
{
return(new DontUnderstandAct());
}
if (!taggedInput.Contains(_label1Tag))
{
return(new NoConnectionToEntityAct(entityInfo(_currentEdgeRepresentant.Item1)));
}
if (!taggedInput.Contains(_label2Tag))
{
return(new NoConnectionToEntityAct(entityInfo(_currentEdgeRepresentant.Item3)));
}
}
if (intextFormat.Replace(_label1Tag, "").Replace(_label2Tag, "").Trim() == "")
{
return(new NotUsefulContinuationAct(startNewFrame()));
}
_currentEdgeHypothesis = intextFormat;
parseCurrentEdgeRepresentant(out string label1, out string requestedRelation, out string edgeHypothesis, out string label2);
return(new EdgeConfirmationRequestAct(label1, intextFormat, label2));
}
private ResponseBase entityLabelHintInput(ParsedUtterance utterance, DialogActBase act)
{
if (act.IsAffirm)
{
return(new ContinueAct());
}
if (act.IsDontKnow || act.IsNegate)
{
return(new NotUsefulContinuationAct(startNewFrame()));
}
if (act.IsAdvice)
{
var advice = act as AdviceAct;
_currentEntityTopic.AddLabelCandidate(advice.Answer.OriginalSentence);
}
HadInformativeInput = utterance.Words.Count() > 2;
CanBeCompleted = true;
// now we are just collecting simple data without any negotiation
return(new UsefulContinuationAct(startNewFrame()));
/*var linkedUtterance = _linker.LinkUtterance(utterance.OriginalSentence);
* if (linkedUtterance.Entities.Count() != 0)
* throw new NotImplementedException("maybe noise, relevant entities, etc");
*
*
* throw new NotImplementedException("parse entities");*/
}
private ParsedUtterance getEquivalenceCandidate(ParsedUtterance question)
{
var bestHypothesis = InputState.QA.GetBestHypothesis(question);
if (bestHypothesis == null)
{
return(null);
}
var score = bestHypothesis.Control.Score;
if (score > AnswerConfidenceThreshold || score < EquivalenceConfidenceThreshold)
{
//no equivalence here
return(null);
}
var equivalenceCandidate = substitute(bestHypothesis.Control.ParsedSentence, question);
if (equivalenceCandidate == null)
{
//we are not able to expose equivalence question
return(null);
}
return(equivalenceCandidate);
}
private ParsedUtterance substitute(ParsedUtterance pattern, ParsedUtterance utterance)
{
var patternNodes = InputState.QA.GetPatternNodes(pattern);
var utteranceNodes = InputState.QA.GetRelatedNodes(utterance).ToArray();
if (patternNodes.Count != utteranceNodes.Length)
{
//semantic equivalence is missing - we won't allow equivalence question
return(null);
}
//substitute every word in pattern, according to utterance
var substitutions = HeuristicQAModule.GetSubstitutions(utteranceNodes, patternNodes, InputState.QA.Graph);
var substitutedWords = new List <string>();
foreach (var patternWord in pattern.Words)
{
var patternNode = InputState.QA.Graph.GetNode(patternWord);
NodeReference substitutedNode;
if (!substitutions.TryGetValue(patternNode, out substitutedNode))
{
//there is no substitution - by default we take node from pattern
substitutedNode = patternNode;
}
substitutedWords.Add(substitutedNode.Data);
}
return(ParsedUtterance.From(substitutedWords));
}
internal void RegisterInput(ParsedUtterance input)
{
UserInput = input;
LastMachineOutput = NextMachineOutput;
NextMachineOutput = null;
HadInformativeInput = false;
}
/// </inheritdoc>
public override ResponseBase Input(ParsedUtterance utterance)
{
//we require enough informative turns only
CanBeCompleted = true;
HadInformativeInput = false;
if (IsDialogClosed)
{
//dialog has been closed - don't say anything
return(null);
}
if (_actualContext == null)
{
throw new NotImplementedException();
}
if (_actualContext.IsComplete)
{
_actualContext = createNewContext();
}
processContext(_actualContext, utterance);
HadInformativeInput = _actualContext.HadInformativeInput;
if (_actualContext.IsComplete)
{
_actualContext = createNewContext();
}
return(_actualContext.NextMachineOutput);
}
internal NodesEnumeration GetPatternNodes(ParsedUtterance sentence)
{
//if (!_questions.ContainsKey(sentence.OriginalSentence))
// throw new KeyNotFoundException("GetPatternNodes: " + sentence.OriginalSentence);
var entry = GetQuestionEntry(sentence);
return(entry.QuestionNodes);
}
internal FeatureCover(FeatureInstance feature, ParsedUtterance utterance)
{
OriginalUtterance = utterance;
_coveredPositions = new bool[feature.MaxOriginPosition + 1];
FeatureInstances = new[] { feature };
indexPositions(feature);
FeatureKey = createFeatureKey();
}
public MappingControl(string substitution, double score, IMappingProvider <T> provider, T value, ParsedUtterance originalSentence)
{
Substitution = substitution;
Score = score;
Value = value;
ParsedSentence = originalSentence;
_provider = provider;
}
private FeatureCover(FeatureCover previousCover, FeatureInstance extendingFeature)
{
OriginalUtterance = previousCover.OriginalUtterance;
_coveredPositions = previousCover._coveredPositions.ToArray();
FeatureInstances = previousCover.FeatureInstances.Concat(new[] { extendingFeature }).OrderBy(f => f.CoveredPositions.First()).ToArray();
indexPositions(extendingFeature);
FeatureKey = createFeatureKey();
}
internal static FeatureInstance CreateSimpleFeatureInstance(ParsedUtterance origin)
{
var coveredPositions = new int[origin.Words.Count()];
for (var i = 0; i < coveredPositions.Length; ++i)
{
coveredPositions[i] = i;
}
return(new FeatureInstance(origin, new SimpleFeature(origin), coveredPositions));
}
internal void AddAnswerHint(QuestionInfo questionInfo, ParsedUtterance utterance)
{
lock (_L_global)
{
//TODO thread safe
var newInfo = questionInfo.WithAnswerHint(utterance);
_questionIndex[questionInfo.Utterance.OriginalSentence] = newInfo;
commitChanges();
}
}
private static IEnumerable <FeatureInstance> createFeatures(ParsedUtterance expression, ComposedGraph graph)
{
var features = new HashSet <FeatureInstance>();
foreach (var generator in _featureGenerators)
{
features.UnionWith(generator.GenerateFeatures(expression, graph));
}
return(features);
}
private bool canBeEquivalent(ParsedUtterance pattern, ParsedUtterance sentence)
{
var sentenceSignature = getSignature(pattern);
var patternSignature = getSignature(sentence);
HashSet <string> disabled;
return(!(
_disabledEquivalencies.TryGetValue(patternSignature, out disabled) &&
disabled.Contains(sentenceSignature)
));
}
private ResponseBase proposeNewQuestion(bool atLeast)
{
_actualQuestion = getNextQuestion();
_isRephrasePhase = true;
_questionRephraseWords.Clear();
CanBeCompleted = true;
AskedDenotations.Clear();
LastDenotationQuestion = DenotationType.None;
return(new RephraseQuestionProposeAct(_actualQuestion, atLeast));
}
public InterpretationsFactory(Dialog.ParsedUtterance parsedQuestion, bool isBasedOnContext, NodeReference correctAnswerNode)
{
if (isBasedOnContext)
{
throw new NotImplementedException();
}
_parsedQuestion = parsedQuestion;
_isBasedOnContext = isBasedOnContext;
CorrectAnswerNode = correctAnswerNode;
ContractedInterpretation = new Interpretation(new[] { new InsertPoolRule(correctAnswerNode) });
}
private bool hasContextIndicator(ParsedUtterance question)
{
foreach (var word in question.Words)
{
if (_contextIndicators.Contains(word))
{
return(true);
}
}
return(false);
}
internal NodeReference GetAnswer(ParsedUtterance expression)
{
var covers = FeatureCover.GetFeatureCovers(expression, Graph);
var allRankedMappings = new List <Ranked <ContextRuleMapping> >();
foreach (var cover in covers)
{
var rankedMappings = _mapping.GetRankedMappings(cover);
allRankedMappings.AddRange(rankedMappings);
}
throw new NotImplementedException();
}
public ResponseBase Input(ParsedUtterance utterance)
{
if (!_isInitialized)
{
throw new InvalidOperationException("Cannot take input before initialization");
}
var inputAct = _factory.GetBestDialogAct(utterance);
_currentState = applyInput(inputAct, _currentState);
//reset expect answer flag after each input.
_currentState = _currentState.SetExpectAnswer(false);
return(applyAction());
}
protected override IEnumerable <FeatureInstance> generateFeatures(ParsedUtterance expression, ComposedGraph graph)
{
var featureInstances = new List <FeatureInstance>();
var wordIndex = 0;
foreach (var word in expression.Words)
{
var feature = new UnigramFeature(word);
var featureInstance = new FeatureInstance(expression, feature, wordIndex);
featureInstances.Add(featureInstance);
++wordIndex;
}
return(featureInstances);
}
internal IEnumerable <MappingControl <T> > FindMapping(ParsedUtterance parsedSentence)
{
var result = new List <MappingControl <T> >();
foreach (var pair in Mapping)
{
var similarity = getSimilarity(pair.Key, parsedSentence);
var control = new MappingControl <T>(similarity.Item1, similarity.Item2, this, pair.Value, pair.Key);
result.Add(control);
}
result.Sort((a, b) => b.Score.CompareTo(a.Score));
return(result);
}
internal IEnumerable <PoolHypothesis> GetSortedHypotheses(ParsedUtterance utterance)
{
var scoredActions = Triggers.FindMapping(utterance);
var availableNodes = GetRelatedNodes(utterance).ToArray();
var result = new List <PoolHypothesis>();
foreach (var scoredAction in scoredActions)
{
var substitutions = GetSubstitutions(availableNodes, scoredAction.Value.RequiredSubstitutions, Graph);
var scoredHypothesis = new PoolHypothesis(substitutions, scoredAction);
result.Add(scoredHypothesis);
}
result.Sort((a, b) => b.Control.Score.CompareTo(a.Control.Score));
return(result);
}
/// <inheritdoc/>
protected override IEnumerable <FeatureInstance> generateFeatures(ParsedUtterance expression, ComposedGraph graph)
{
var featureInstances = new List <FeatureInstance>();
var wordIndex = 0;
foreach (var word in expression.Words)
{
if (graph.HasEvidence(word))
{
var nodeFeature = new NodeFeature(wordIndex);
var instance = new FeatureInstance(expression, nodeFeature, wordIndex);
featureInstances.Add(instance);
}
++wordIndex;
}
return(featureInstances);
}
/// </inheritdoc>
public override ResponseBase Input(ParsedUtterance utterance)
{
HadInformativeInput = false;
CanBeCompleted = false;
var act = Factory.GetBestDialogAct(utterance);
switch (_currentState)
{
case NavigationDialogState.ExpectsEntityLabelHint:
return(entityLabelHintInput(utterance, act));
case NavigationDialogState.ExpectsEdgeHint:
return(edgeHintInput(utterance, act));
default:
throw new NotImplementedException("Unknown state");
}
}
internal QuestionEntry GetQuestionEntry(ParsedUtterance question)
{
QuestionEntry entry;
if (!_questions.TryGetValue(question.OriginalSentence, out entry))
{
_questions[question.OriginalSentence] = entry = new QuestionEntry(question.OriginalSentence, Graph);
var pattern = getPatternQuestion(entry);
List <QuestionEntry> patternQuestions;
if (!_questionsPatternIndex.TryGetValue(pattern, out patternQuestions))
{
_questionsPatternIndex[pattern] = patternQuestions = new List <QuestionEntry>();
}
patternQuestions.Add(entry);
}
return(entry);
}
private string substitute(ParsedUtterance pattern, ParsedUtterance utterance)
{
var patternNodes = _context.QuestionAnsweringModule.GetPatternNodes(pattern);
var utteranceNodes = _context.QuestionAnsweringModule.GetRelatedNodes(utterance).ToArray();
var substitutions = HeuristicQAModule.GetSubstitutions(utteranceNodes, patternNodes, _context.Graph);
var result = repairSpelling(pattern.OriginalSentence);
foreach (var patternNode in patternNodes)
{
NodeReference substitution;
if (!substitutions.TryGetValue(patternNode, out substitution))
{
substitution = patternNode;
}
result = result.Replace(patternNode.Data.ToString(), substitution.Data.ToString());
}
return(result);
}
private string getImportantDifferenceWord(ParsedUtterance utterance1, ParsedUtterance utterance2)
{
var utteranceWords1 = new HashSet <string>(utterance1.Words);
var utteranceWords2 = new HashSet <string>(utterance2.Words);
var intersectionSyntacticWords = utteranceWords1.Intersect(utteranceWords2).ToArray();
var differenceWords = utteranceWords1.Union(utteranceWords2).Except(intersectionSyntacticWords).Where(w => !InputState.QA.Graph.HasEvidence(w)).ToArray();
//find best difference word
string longestWord = null;
foreach (var word in differenceWords)
{
if (longestWord == null || longestWord.Length < word.Length)
{
//TODO this can be better when considering word distribution probability
longestWord = word;
}
}
return(longestWord);
}
public override ResponseBase Input(ParsedUtterance utterance)
{
//Database.DebugTrigger(849);
string response;
try
{
_inputCount += 1;
if (_inputCount > 10)
{
response = "[NOTE] Too much inputs. The task should be easier - use simple phrases. Type reset to start the dialog from beginning.";
}
else
{
var turnStartNode = _agent.LastBestNode;
response = _agent.Input(utterance.OriginalSentence);
var turnEndNode = _agent.LastBestNode;
if (hasInformativeInput())
{
_hadInformativeInput = true;
}
if (_exportKnowledge)
{
voteForExportedEvents(turnStartNode, turnEndNode);
}
}
}
catch (Exception ex)
{
Console.WriteLine(ex);
File.AppendAllText("phrase_agent_manager.exceptions", DateTime.Now + "\n" + ex.ToString() + "\n\n\n\n");
response = "[ERROR] The bot encountered an unexpected error - type reset and try to do the dialog differently.";
}
return(new SimpleResponse(response));
}