public void TestTrigramLanguageModelCreationFromText() { var ngramSize = 3; var languageModel = new NGramLanguageModel(ngramSize); var stream = Tests.OpenFile("/opennlp/tools/languagemodel/sentences.txt", Encoding.UTF8); string line; while ((line = stream.ReadLine()) != null) { var list = new List <string>(line.Split(new[] { ' ' }, StringSplitOptions.None)); var generatedStrings = NGramGenerator.Generate(list, ngramSize, " "); foreach (var generatedString in generatedStrings) { var tokens = generatedString.Split(new[] { ' ' }, StringSplitOptions.None); if (tokens.Length > 0) { languageModel.Add(new StringList(tokens), 1, ngramSize); } } } var predited = languageModel.PredictNextTokens(new StringList("neural", "network", "language")); Assert.That(predited, Is.EqualTo(new StringList("models"))); var p1 = languageModel.CalculateProbability(new StringList("neural", "network", "language", "models")); var p2 = languageModel.CalculateProbability(new StringList("neural", "network", "language", "model")); Assert.That(p1, Is.GreaterThan(p2)); }
private Tuple <List <string>, List <string> > GetColumnsSemiGuaranteed(ICompiledGram compiled, ICompiledGram simpleCompiled) { List <string> columns; List <string> simplified; if (simplifiedGram == null) { columns = NGramGenerator.Generate(compiled, startInput, size, includeStart: false); simplified = LevelParser.BreakColumnsIntoSimplifiedTokens( columns, game); } else { simplified = NGramGenerator.GenerateBestAttempt( simpleCompiled, LevelParser.BreakColumnsIntoSimplifiedTokens(startInput, game), size, maxAttempts); Games localGame = game; columns = NGramGenerator.GenerateRestricted( compiled, startInput, simplified, (inColumn) => { return(LevelParser.ClassifyColumn(inColumn, localGame)); }, includeStart: false); } return(new Tuple <List <string>, List <string> >(columns, simplified)); }
private bool GenerateLevel() { List <List <char> > level = new List <List <char> >(); if (blackBoard.ConfigUI.Config.UsingSimplifiedNGram) { ICompiledGram compiledGram = simpleGrammar.Compile(); int levelIndex = levelColumns.RandomIndex(); List <string> simpleInput = simplifiedLevelColumns[levelIndex].GetRange (0, blackBoard.ConfigUI.Config.N + 7); blackBoard.LevelColumns = levelColumns[levelIndex].GetRange( 0, blackBoard.ConfigUI.Config.N + 7); blackBoard.SimpleLevelColumns = NGramGenerator.Generate( compiledGram, simpleInput, blackBoard.ConfigUI.Config.LevelSize); compiledGram = grammar.Compile(); blackBoard.LevelColumns = NGramGenerator.GenerateRestricted( compiledGram, blackBoard.LevelColumns, blackBoard.SimpleLevelColumns, (inColumn) => { return(LevelParser.ClassifyColumn( inColumn, blackBoard.ConfigUI.Config.Game)); }); } else { ICompiledGram compiledGram = grammar.Compile(); blackBoard.LevelColumns = NGramGenerator.Generate( compiledGram, levelColumns.RandomValue().GetRange(0, blackBoard.ConfigUI.Config.N + 7), blackBoard.ConfigUI.Config.LevelSize); } bool generationWorked = blackBoard.LevelColumns != null; if (generationWorked) { foreach (string column in blackBoard.LevelColumns) { level.Add(new List <char>(column)); } // add ending column to the level char flagChar = Tile.playerOneFinish.ToChar(); List <char> endingColumn = new List <char>(); for (int i = 0; i < level[0].Count; ++i) { endingColumn.Add(flagChar); } level.Add(endingColumn); blackBoard.LevelInfo = LevelLoader.Build(level, blackBoard.Tilemap, blackBoard.CameraFollow); } return(generationWorked); }