public void TestTrigramLanguageModelCreationFromText()
{
var ngramSize = 3;
var languageModel = new NGramLanguageModel(ngramSize);
var stream = Tests.OpenFile("/opennlp/tools/languagemodel/sentences.txt", Encoding.UTF8);
string line;
while ((line = stream.ReadLine()) != null)
{
var list = new List <string>(line.Split(new[] { ' ' }, StringSplitOptions.None));
var generatedStrings = NGramGenerator.Generate(list, ngramSize, " ");
foreach (var generatedString in generatedStrings)
{
var tokens = generatedString.Split(new[] { ' ' }, StringSplitOptions.None);
if (tokens.Length > 0)
{
languageModel.Add(new StringList(tokens), 1, ngramSize);
}
}
}
var predited = languageModel.PredictNextTokens(new StringList("neural", "network", "language"));
Assert.That(predited, Is.EqualTo(new StringList("models")));
var p1 = languageModel.CalculateProbability(new StringList("neural", "network", "language", "models"));
var p2 = languageModel.CalculateProbability(new StringList("neural", "network", "language", "model"));
Assert.That(p1, Is.GreaterThan(p2));
}
private Tuple <List <string>, List <string> > GetColumnsSemiGuaranteed(ICompiledGram compiled, ICompiledGram simpleCompiled)
{
List <string> columns;
List <string> simplified;
if (simplifiedGram == null)
{
columns = NGramGenerator.Generate(compiled, startInput, size, includeStart: false);
simplified = LevelParser.BreakColumnsIntoSimplifiedTokens(
columns,
game);
}
else
{
simplified = NGramGenerator.GenerateBestAttempt(
simpleCompiled,
LevelParser.BreakColumnsIntoSimplifiedTokens(startInput, game),
size,
maxAttempts);
Games localGame = game;
columns = NGramGenerator.GenerateRestricted(
compiled,
startInput,
simplified,
(inColumn) =>
{
return(LevelParser.ClassifyColumn(inColumn, localGame));
},
includeStart: false);
}
return(new Tuple <List <string>, List <string> >(columns, simplified));
}
private bool GenerateLevel()
{
List <List <char> > level = new List <List <char> >();
if (blackBoard.ConfigUI.Config.UsingSimplifiedNGram)
{
ICompiledGram compiledGram = simpleGrammar.Compile();
int levelIndex = levelColumns.RandomIndex();
List <string> simpleInput = simplifiedLevelColumns[levelIndex].GetRange
(0,
blackBoard.ConfigUI.Config.N + 7);
blackBoard.LevelColumns = levelColumns[levelIndex].GetRange(
0,
blackBoard.ConfigUI.Config.N + 7);
blackBoard.SimpleLevelColumns = NGramGenerator.Generate(
compiledGram,
simpleInput,
blackBoard.ConfigUI.Config.LevelSize);
compiledGram = grammar.Compile();
blackBoard.LevelColumns = NGramGenerator.GenerateRestricted(
compiledGram,
blackBoard.LevelColumns,
blackBoard.SimpleLevelColumns,
(inColumn) =>
{
return(LevelParser.ClassifyColumn(
inColumn,
blackBoard.ConfigUI.Config.Game));
});
}
else
{
ICompiledGram compiledGram = grammar.Compile();
blackBoard.LevelColumns = NGramGenerator.Generate(
compiledGram,
levelColumns.RandomValue().GetRange(0, blackBoard.ConfigUI.Config.N + 7),
blackBoard.ConfigUI.Config.LevelSize);
}
bool generationWorked = blackBoard.LevelColumns != null;
if (generationWorked)
{
foreach (string column in blackBoard.LevelColumns)
{
level.Add(new List <char>(column));
}
// add ending column to the level
char flagChar = Tile.playerOneFinish.ToChar();
List <char> endingColumn = new List <char>();
for (int i = 0; i < level[0].Count; ++i)
{
endingColumn.Add(flagChar);
}
level.Add(endingColumn);
blackBoard.LevelInfo = LevelLoader.Build(level, blackBoard.Tilemap, blackBoard.CameraFollow);
}
return(generationWorked);
}