public void Learn(TPhrase phrase)
{
Logger.Info($"Learning phrase: '{phrase}'");
if (phrase == null || phrase.Equals(default(TPhrase)))
{
return;
}
// Ignore particularly short phrases
if (SplitTokens(phrase).Count() < Level)
{
Logger.Info($"Phrase {phrase} too short - skipped");
return;
}
// Add it to the source lines so we can ignore it
// when learning in future
if (!SourcePhrases.Contains(phrase))
{
Logger.Debug($"Adding phrase {phrase} to source lines");
SourcePhrases.Add(phrase);
}
// Split the sentence to an array of words
var tokens = SplitTokens(phrase).ToArray();
LearnTokens(tokens);
var lastCol = new List <TUnigram>();
for (var j = Level; j > 0; j--)
{
TUnigram previous;
try
{
previous = tokens[tokens.Length - j];
Logger.Debug($"Adding TGram ({typeof(TUnigram)}) {previous} to lastCol");
lastCol.Add(previous);
}
catch (IndexOutOfRangeException e)
{
Logger.Warn($"Caught an exception: {e}");
previous = GetPrepadUnigram();
lastCol.Add(previous);
}
}
Logger.Debug($"Reached final key for phrase {phrase}");
var finalKey = new NgramContainer <TUnigram>(lastCol.ToArray());
Chain.AddOrCreate(finalKey, GetTerminatorUnigram());
}
public void Learn(TPhrase phrase)
{
if (phrase == null || phrase.Equals(default(TPhrase)))
{
return;
}
// Ignore particularly short phrases
if (SplitTokens(phrase).Count() < Level)
{
return;
}
// Add it to the source lines so we can ignore it
// when learning in future
if (!SourcePhrases.Contains(phrase))
{
SourcePhrases.Add(phrase);
}
// Split the sentence to an array of words
var tokens = SplitTokens(phrase).ToArray();
LearnTokens(tokens);
var lastCol = new List <TUnigram>();
for (var j = Level; j > 0; j--)
{
TUnigram previous;
try
{
previous = tokens[tokens.Length - j];
lastCol.Add(previous);
}
catch (IndexOutOfRangeException e)
{
previous = GetPrepadUnigram();
lastCol.Add(previous);
}
}
var finalKey = new NgramContainer <TUnigram>(lastCol.ToArray());
Chain.AddOrCreate(finalKey, GetTerminatorUnigram());
}
public void Learn(TPhrase phrase)
{
// Logger.Info($"Learning phrase: '{phrase}'");
if (phrase == null || phrase.Equals(default(TPhrase)))
{
return;
}
// Ignore particularly short phrases
if (SplitTokens(phrase).Count() < Level)
{
// Logger.Info($"Phrase {phrase} too short - skipped");
return;
}
// Add it to the source lines so we can ignore it
// when learning in future
if (!SourcePhrases.Contains(phrase))
{
// Logger.Debug($"Adding phrase {phrase} to source lines");
SourcePhrases.Add(phrase);
}
// Split the sentence to an array of words
var tokens = SplitTokens(phrase).ToArray();
LearnTokens(tokens);
var lastCol = new List <TUnigram>();
for (var j = Level; j > 0; j--)
{
TUnigram previous;
try
{
previous = tokens[^ j];