/// <summary> /// Runs the comparison between TrueSkill and Random models with draws included. /// </summary> /// <typeparam name="TGame">The type of the game.</typeparam> /// <param name="name">The name.</param> /// <param name="modelFunc">The model function.</param> /// <param name="inputs">The inputs.</param> /// <returns>The experiment collection.</returns> private OnlineExperimentComparison <TGame> RunComparison <TGame>(string name, Func <IModelParameters, bool, bool, IModel> modelFunc, Inputs <TGame> inputs) where TGame : Game { var randomParameters = new RandomModelParameters { IncludeDraws = true, EmpiricalDrawProportion = inputs.DrawProportion }; var trueSkillParameters = inputs.TrueSkillParameters; trueSkillParameters.DynamicsVariance = 0; // Remove games with draws. var gamesWithoutDraws = inputs.Games.Where(ia => ia is TeamGame && (ia as TeamGame).Outcome != TeamMatchOutcome.Draw).ToList(); var collection = new OnlineExperimentComparison <TGame>( new OnlineExperiment(modelFunc, trueSkillParameters) { Players = inputs.Players, Priors = inputs.TrueSkillPriors, SkillPrior = inputs.SkillPrior, Name = "TrueSkill" }, new OnlineExperiment(ia => new RandomModel(ia), randomParameters) { Players = inputs.Players, Priors = inputs.TrueSkillPriors, SkillPrior = inputs.SkillPrior, Name = "Random" }) { Name = name }; collection.AnnounceAndRunAll(gamesWithoutDraws, gamesWithoutDraws.Count); LogProbs[collection.Name] = collection.CumulativeNegativeLogProbOfTruth; return(collection); }
/// <summary> /// Head to head. /// </summary> public void HeadToHead() { AnnounceExperiment("HeadToHead"); var inputs = FileUtils.Load <Inputs <TwoPlayerGame> >(DataPath, "Halo2-HeadToHead"); // We know that there are issues with players Gamer01266 and Gamer00296 inputs.Games = new KeyedCollectionWithFunc <string, TwoPlayerGame>( inputs.Games.Where(ia => !ia.Players.Contains("Gamer01266") && !ia.Players.Contains("Gamer00296")), ia => ia.Id); inputs.Players.Remove("Gamer01266"); inputs.Players.Remove("Gamer00296"); var datasetSummary = new DatasetSummary { Name = "Head to Head", NumberOfGames = inputs.NumberOfGames, NumberOfPlayers = inputs.NumberOfPlayers, TeamsPerGame = 2, PlayersPerTeam = 1, PlayersPerGame = 2 }; outputter.Out(inputs, Contents.S3ASolutionExpectationPropagation.NumberedName, "Halo2-HeadToHead", "Dataset" ); outputter.Out(datasetSummary, Contents.S3ASolutionExpectationPropagation.NumberedName, "Halo2-HeadToHead", "Dataset summary"); outputter.Out(inputs, Contents.S4ExtensionsToTheCoreModel.NumberedName, "Halo2-HeadToHead", "Dataset" ); outputter.Out(datasetSummary, Contents.S4ExtensionsToTheCoreModel.NumberedName, "Halo2-HeadToHead", "Dataset summary"); // Get games of top two players GetTopTwo(inputs, out var topTwoPlayers, out var topTwoGames); // Get games without draws var gamesWithoutDraws = inputs.Games.Where(ia => ia.Outcome != MatchOutcome.Draw).ToList(); Console.WriteLine(@"Number of games without draws in Halo2 head to head: " + gamesWithoutDraws.Count); Console.WriteLine(@"Number of players in Halo2 head to head: " + inputs.Players.Count); var alphas = new[] { 0.07, 0.35 }.Select(ia => ia * inputs.Beta).ToArray(); var randomParameters1 = new RandomModelParameters { IncludeDraws = false }; var randomParameters2 = alphas.Select( alpha => new RandomModelParameters { IncludeDraws = true, EmpiricalDrawProportion = inputs.DrawProportion }).ToArray(); var trueSkillParameters = inputs.TrueSkillParameters; trueSkillParameters.DynamicsVariance = 0; var withoutDraws = new OnlineExperimentComparison <TwoPlayerGame>( new OnlineExperiment(ia => new TwoPlayer(ia, ShowFactorGraph), trueSkillParameters) { Players = inputs.Players, Priors = inputs.TrueSkillPriors, SkillPrior = inputs.SkillPrior, Name = "TrueSkill" }, new OnlineExperiment(ia => new RandomModel(ia), randomParameters1) { Players = inputs.Players, Priors = inputs.TrueSkillPriors, SkillPrior = inputs.SkillPrior, Name = "Random" }) { Name = "HeadToHead" }; withoutDraws.AnnounceAndRunAll(gamesWithoutDraws, gamesWithoutDraws.Count); var withDraws = new OnlineExperimentComparison <TwoPlayerGame>( new OnlineExperiment(ia => new TwoPlayerWithDraws(ia, ShowFactorGraph), trueSkillParameters) { Players = inputs.Players, Priors = inputs.TrueSkillPriors, SkillPrior = inputs.SkillPrior, Name = "TrueSkill" }, new OnlineExperiment(ia => new RandomModel(ia), randomParameters2[0]) { Players = inputs.Players, Priors = inputs.TrueSkillPriors, SkillPrior = inputs.SkillPrior, Name = "Random" }) { Name = "HeadToHeadWithDraws" }; withDraws.AnnounceAndRunAll(inputs); var topTwo = new OnlineExperimentComparison <TwoPlayerGame>( new OnlineExperiment(ia => new TwoPlayerWithDraws(ia, ShowFactorGraph), trueSkillParameters) { Players = topTwoPlayers, Priors = inputs.TrueSkillPriors, SkillPrior = inputs.SkillPrior, Name = "TrueSkill" }); topTwo.AnnounceAndRunAll(topTwoGames, topTwoGames.Count); outputter.Out( withoutDraws, Contents.S3ASolutionExpectationPropagation.NumberedName, "Halo2-HeadToHead", "HeadToHead"); outputter.Out( withoutDraws, Contents.S4ExtensionsToTheCoreModel.NumberedName, "Halo2-HeadToHead", "HeadToHead"); outputter.Out( withDraws, Contents.S4ExtensionsToTheCoreModel.NumberedName, "Halo2-HeadToHead", "HeadToHeadWithDraws"); var headToHeadTopTwo = new Dictionary <string, object> { { "Players", topTwoPlayers }, { "Games", topTwoGames }, { "1v1", topTwo.GetTrajectories(OnlineExperiment.GetTopNPlayersBySkill, Utils.GetMeanAndStandardDeviation, 2) .Where(ia => topTwoPlayers.Contains(ia.Key.Split(' ')[0]) && !ia.Key.Contains("Random")).ToDictionary() }, { "1vAll", withDraws.GetTrajectories(OnlineExperiment.GetTopNPlayersBySkill, Utils.GetMeanAndStandardDeviation, int.MaxValue) .Where(ia => topTwoPlayers.Contains(ia.Key.Split(' ')[0]) && !ia.Key.Contains("Random")).ToDictionary() } }; outputter.Out( headToHeadTopTwo, Contents.S4ExtensionsToTheCoreModel.NumberedName, "Halo2-HeadToHead", "HeadToHeadTopTwo"); }