/// <summary>
/// Runs the comparison between TrueSkill and Random models with draws included.
/// </summary>
/// <typeparam name="TGame">The type of the game.</typeparam>
/// <param name="name">The name.</param>
/// <param name="modelFunc">The model function.</param>
/// <param name="inputs">The inputs.</param>
/// <returns>The experiment collection.</returns>
private OnlineExperimentComparison <TGame> RunComparison <TGame>(string name, Func <IModelParameters, bool, bool, IModel> modelFunc, Inputs <TGame> inputs)
where TGame : Game
{
var randomParameters = new RandomModelParameters
{
IncludeDraws = true,
EmpiricalDrawProportion = inputs.DrawProportion
};
var trueSkillParameters = inputs.TrueSkillParameters;
trueSkillParameters.DynamicsVariance = 0;
// Remove games with draws.
var gamesWithoutDraws = inputs.Games.Where(ia => ia is TeamGame && (ia as TeamGame).Outcome != TeamMatchOutcome.Draw).ToList();
var collection =
new OnlineExperimentComparison <TGame>(
new OnlineExperiment(modelFunc, trueSkillParameters)
{
Players = inputs.Players,
Priors = inputs.TrueSkillPriors,
SkillPrior = inputs.SkillPrior,
Name = "TrueSkill"
},
new OnlineExperiment(ia => new RandomModel(ia), randomParameters)
{
Players = inputs.Players,
Priors = inputs.TrueSkillPriors,
SkillPrior = inputs.SkillPrior,
Name = "Random"
})
{
Name = name
};
collection.AnnounceAndRunAll(gamesWithoutDraws, gamesWithoutDraws.Count);
LogProbs[collection.Name] = collection.CumulativeNegativeLogProbOfTruth;
return(collection);
}
/// <summary>
/// Demonstration of Dynamics.
/// </summary>
/// <param name="trueBeta">The true beta.</param>
/// <param name="beta">The beta.</param>
/// <param name="stepSize">Size of the step.</param>
public void DynamicsDemo(double trueBeta, double beta, double stepSize)
{
var randomNames = new RandomNameGenerator(0, int.MaxValue, true);
var players = new HashSet <string>(randomNames.Take(100));
string improver = players.First();
const double Mu = 125;
const double Sigma = 10;
var skillDistribution = new Gaussian(Mu, Sigma * Sigma);
const double Gamma = 0.4;
const double SkillStart = 110;
const double SkillMax = 140;
var inputs1 = new Inputs <TwoPlayerGame> {
Mu = Mu, Sigma = Sigma, Beta = beta, Gamma = 0
};
var inputs2 = new Inputs <TwoPlayerGame> {
Mu = Mu, Sigma = Sigma, Beta = beta, Gamma = Gamma
};
double truePerformanceVariance = trueBeta * trueBeta;
// number of games
const int NumberOfGames = 25000;
var skill1 = Gaussian.PointMass(SkillStart);
var random = new Random(0);
var initialSkills = players.ToDictionary(ia => ia, ia => Gaussian.PointMass(skillDistribution.Sample()));
initialSkills[improver] = skill1;
var trueSkills = new Dictionary <string, List <double> > {
{ improver, new List <double>() }
};
for (int i = 0; i < NumberOfGames; i++)
{
// Pick two players to play game
var gamePlayers = players.OrderBy(ia => random.Next()).Take(2).ToList();
// update improver's skill
var truth = new Marginals {
Skills = new Dictionary <string, Gaussian>(initialSkills)
};
truth.Skills[improver] = skill1;
// Generate game according to truth
var game = TwoPlayerVaryingSkills.Sample(truth, gamePlayers, truePerformanceVariance);
inputs1.Games.Add(game);
inputs2.Games.Add(game);
if (gamePlayers.Contains(improver))
{
trueSkills[improver].Add(skill1.Point);
// Update skill
if (trueSkills[improver].Count > 100 && (trueSkills[improver].Count % 100) >= 90 && skill1.Point < SkillMax)
{
skill1 = Gaussian.PointMass(skill1.Point + stepSize);
}
}
}
var experiment1 = new OnlineExperiment(ia => new TwoPlayerVaryingSkills(ia, ShowFactorGraph), inputs1.TrueSkillParameters)
{
Players = players,
Priors = inputs1.TrueSkillPriors,
Name = "Fixed skill model"
};
var experiment2 = new OnlineExperiment(ia => new TwoPlayerVaryingSkills(ia, ShowFactorGraph), inputs2.TrueSkillParameters)
{
Players = players,
Priors = inputs2.TrueSkillPriors,
Name = "Varying skill model"
};
AnnounceExperiment(experiment1.Name);
experiment1.Run(inputs1.Games, inputs1.Games.Count, false);
AnnounceExperiment(experiment2.Name);
experiment2.Run(inputs2.Games, inputs2.Games.Count, false);
var experiments1 = new OnlineExperimentComparison <TwoPlayerGame>(experiment1)
{
Truth = trueSkills
};
var experiments2 = new OnlineExperimentComparison <TwoPlayerGame>(experiment2)
{
Truth = trueSkills
};
var initialSkillsTable =
initialSkills.OrderByDescending(ia => ia.Key)
.ToDictionaryForTable <KeyValuePair <string, Gaussian>, string, double, double>(
ia => ia.Key,
new Dictionary <string, Func <KeyValuePair <string, Gaussian>, double> > {
{ "Skill", g => g.Value.Point }
},
false,
null,
"Player");
var collection = new OnlineExperimentComparison <TwoPlayerGame>(experiment1, experiment2)
{
Truth = trueSkills
};
var dynamicsDemo = new Dictionary <string, object>
{
{ "InitialSkills", initialSkillsTable },
{ "SampledInputs", inputs1 },
{
"Trajectories",
new[]
{
experiments1.PlayerTrajectoriesWithSigmaAndTruth,
experiments2.PlayerTrajectoriesWithSigmaAndTruth,
collection.PlayerTrajectoriesWithSigmaAndTruth
}
}
};
outputter.Out(dynamicsDemo,
Contents.S5AllowingTheSkillsToVary.NumberedName,
"DynamicsDemo");
}
/// <summary>
/// Head to head.
/// </summary>
public void HeadToHead()
{
AnnounceExperiment("HeadToHead");
var inputs = FileUtils.Load <Inputs <TwoPlayerGame> >(DataPath, "Halo2-HeadToHead");
// We know that there are issues with players Gamer01266 and Gamer00296
inputs.Games =
new KeyedCollectionWithFunc <string, TwoPlayerGame>(
inputs.Games.Where(ia => !ia.Players.Contains("Gamer01266") && !ia.Players.Contains("Gamer00296")),
ia => ia.Id);
inputs.Players.Remove("Gamer01266");
inputs.Players.Remove("Gamer00296");
var datasetSummary = new DatasetSummary
{
Name = "Head to Head",
NumberOfGames = inputs.NumberOfGames,
NumberOfPlayers = inputs.NumberOfPlayers,
TeamsPerGame = 2,
PlayersPerTeam = 1,
PlayersPerGame = 2
};
outputter.Out(inputs,
Contents.S3ASolutionExpectationPropagation.NumberedName,
"Halo2-HeadToHead",
"Dataset"
);
outputter.Out(datasetSummary,
Contents.S3ASolutionExpectationPropagation.NumberedName,
"Halo2-HeadToHead",
"Dataset summary");
outputter.Out(inputs,
Contents.S4ExtensionsToTheCoreModel.NumberedName,
"Halo2-HeadToHead",
"Dataset"
);
outputter.Out(datasetSummary,
Contents.S4ExtensionsToTheCoreModel.NumberedName,
"Halo2-HeadToHead",
"Dataset summary");
// Get games of top two players
GetTopTwo(inputs, out var topTwoPlayers, out var topTwoGames);
// Get games without draws
var gamesWithoutDraws = inputs.Games.Where(ia => ia.Outcome != MatchOutcome.Draw).ToList();
Console.WriteLine(@"Number of games without draws in Halo2 head to head: " + gamesWithoutDraws.Count);
Console.WriteLine(@"Number of players in Halo2 head to head: " + inputs.Players.Count);
var alphas = new[] { 0.07, 0.35 }.Select(ia => ia * inputs.Beta).ToArray();
var randomParameters1 = new RandomModelParameters {
IncludeDraws = false
};
var randomParameters2 =
alphas.Select(
alpha =>
new RandomModelParameters
{
IncludeDraws = true,
EmpiricalDrawProportion = inputs.DrawProportion
}).ToArray();
var trueSkillParameters = inputs.TrueSkillParameters;
trueSkillParameters.DynamicsVariance = 0;
var withoutDraws =
new OnlineExperimentComparison <TwoPlayerGame>(
new OnlineExperiment(ia => new TwoPlayer(ia, ShowFactorGraph), trueSkillParameters)
{
Players = inputs.Players,
Priors = inputs.TrueSkillPriors,
SkillPrior = inputs.SkillPrior,
Name = "TrueSkill"
},
new OnlineExperiment(ia => new RandomModel(ia), randomParameters1)
{
Players = inputs.Players,
Priors = inputs.TrueSkillPriors,
SkillPrior = inputs.SkillPrior,
Name = "Random"
})
{
Name = "HeadToHead"
};
withoutDraws.AnnounceAndRunAll(gamesWithoutDraws, gamesWithoutDraws.Count);
var withDraws =
new OnlineExperimentComparison <TwoPlayerGame>(
new OnlineExperiment(ia => new TwoPlayerWithDraws(ia, ShowFactorGraph), trueSkillParameters)
{
Players = inputs.Players,
Priors = inputs.TrueSkillPriors,
SkillPrior = inputs.SkillPrior,
Name = "TrueSkill"
},
new OnlineExperiment(ia => new RandomModel(ia), randomParameters2[0])
{
Players = inputs.Players,
Priors = inputs.TrueSkillPriors,
SkillPrior = inputs.SkillPrior,
Name = "Random"
})
{
Name = "HeadToHeadWithDraws"
};
withDraws.AnnounceAndRunAll(inputs);
var topTwo =
new OnlineExperimentComparison <TwoPlayerGame>(
new OnlineExperiment(ia => new TwoPlayerWithDraws(ia, ShowFactorGraph), trueSkillParameters)
{
Players = topTwoPlayers,
Priors = inputs.TrueSkillPriors,
SkillPrior = inputs.SkillPrior,
Name = "TrueSkill"
});
topTwo.AnnounceAndRunAll(topTwoGames, topTwoGames.Count);
outputter.Out(
withoutDraws,
Contents.S3ASolutionExpectationPropagation.NumberedName,
"Halo2-HeadToHead",
"HeadToHead");
outputter.Out(
withoutDraws,
Contents.S4ExtensionsToTheCoreModel.NumberedName,
"Halo2-HeadToHead",
"HeadToHead");
outputter.Out(
withDraws,
Contents.S4ExtensionsToTheCoreModel.NumberedName,
"Halo2-HeadToHead",
"HeadToHeadWithDraws");
var headToHeadTopTwo = new Dictionary <string, object>
{
{ "Players", topTwoPlayers },
{ "Games", topTwoGames },
{ "1v1",
topTwo.GetTrajectories(OnlineExperiment.GetTopNPlayersBySkill, Utils.GetMeanAndStandardDeviation, 2)
.Where(ia => topTwoPlayers.Contains(ia.Key.Split(' ')[0]) && !ia.Key.Contains("Random")).ToDictionary() },
{ "1vAll",
withDraws.GetTrajectories(OnlineExperiment.GetTopNPlayersBySkill, Utils.GetMeanAndStandardDeviation, int.MaxValue)
.Where(ia => topTwoPlayers.Contains(ia.Key.Split(' ')[0]) && !ia.Key.Contains("Random")).ToDictionary() }
};
outputter.Out(
headToHeadTopTwo,
Contents.S4ExtensionsToTheCoreModel.NumberedName,
"Halo2-HeadToHead",
"HeadToHeadTopTwo");
}
