void CreateEvaluator()
{
EvaluatorDef = new NNEvaluatorDef(NetworkSpec.ComboType, NetworkSpec.NetDefs,
DeviceSpec.ComboType, DeviceSpec.Devices);
OutStream.WriteLine($"Network evaluation configured to use: {EvaluatorDef}");
}
/// <summary>
/// Constructor.
/// </summary>
/// <param name="id">descriptive identifier</param>
/// <param name="paramsNN">specification of the neural network to be used</param>
/// <param name="forceDisableSmartPruning"></param>
/// <param name="emulateCeresSettings"></param>
/// <param name="searchParams"></param>
/// <param name="selectParams"></param>
/// <param name="uciSetOptionCommands"></param>
/// <param name="callback"></param>
/// <param name="overrideEXE">optinally the full name of the executable file(otherwise looks for Ceres executable in working directory)</param>
public GameEngineCeresUCI(string id,
NNEvaluatorDef paramsNN,
bool forceDisableSmartPruning = false,
bool emulateCeresSettings = false,
ParamsSearch searchParams = null, ParamsSelect selectParams = null,
string[] uciSetOptionCommands = null,
ProgressCallback callback = null,
string overrideEXE = null)
: base(id, GetExecutableFN(overrideEXE), null, null, null, uciSetOptionCommands, callback, false, ExtraArgsForEvaluator(paramsNN))
{
// TODO: support some limited emulation of options
if (searchParams != null || selectParams != null)
{
throw new NotSupportedException("Customized searchParams and selectParams not yet supported");
}
if (paramsNN == null)
{
throw new ArgumentNullException(nameof(paramsNN));
}
if (forceDisableSmartPruning)
{
throw new NotImplementedException(nameof(forceDisableSmartPruning));
}
if (emulateCeresSettings)
{
throw new NotImplementedException(nameof(emulateCeresSettings));
}
}
/// <summary>
/// Experimental sample code of runnings suites via the API.
/// </summary>
public static void RunSuiteTest()
{
const int PARALLELISM = 1;
string deviceSuffix = PARALLELISM > 1 ? ":POOLED" : "";
NNEvaluatorDef evalDef1 = NNEvaluatorDefFactory.FromSpecification("LC0:j92-280", $"GPU:1{deviceSuffix}");
NNEvaluatorDef evalDef2 = NNEvaluatorDefFactory.FromSpecification("LC0:66733", $"GPU:1{deviceSuffix}");
SearchLimit limit = SearchLimit.NodesPerMove(10_000);
string[] extraUCI = new string[] { "setoption name Contempt value 5000" };
GameEngineDef ged1 = new GameEngineDefCeres("Ceres1", evalDef1);
GameEngineDef ged2 = new GameEngineDefCeres("Ceres1", evalDef2);
GameEngineUCISpec geSF = new GameEngineUCISpec("SF12", @"\\synology\dev\chess\engines\stockfish_20090216_x64_avx2.exe",
32, 2048, CeresUserSettingsManager.Settings.DirTablebases,
uciSetOptionCommands: extraUCI);
EnginePlayerDef ceresEngineDef1 = new EnginePlayerDef(ged1, limit);
EnginePlayerDef ceresEngineDef2 = new EnginePlayerDef(ged2, limit);
GameEngineDefUCI sf12EngineDef = new GameEngineDefUCI("SF12", geSF);
EnginePlayerDef sfEngine = new EnginePlayerDef(sf12EngineDef, limit * 875);
SuiteTestDef def = new SuiteTestDef("Test1", @"\\synology\dev\chess\data\epd\ERET_VESELY203.epd",
ceresEngineDef1, ceresEngineDef2, sfEngine);
def.MaxNumPositions = 1500;
SuiteTestRunner ser = new SuiteTestRunner(def);
ser.Run(PARALLELISM, true);
}
static GameEngine GetEngine(GameEngineUCISpec engineSpec, string suffix,
NNEvaluatorDef evaluatorDef,
ParamsSearch paramsSearch, ParamsSelect paramsSelect, IManagerGameLimit timeManager)
{
bool resetMovesBetweenMoves = !paramsSearch.TreeReuseEnabled;
bool enableTranpsositions = paramsSearch.Execution.TranspositionMode != TranspositionMode.None;
// Create requested type of engine
if (engineSpec == null)
{
return(new GameEngineCeresInProcess("Ceres_" + suffix, evaluatorDef, paramsSearch, paramsSelect, timeManager, null));
}
else if (engineSpec.Name == "LC0")
{
if (evaluatorDef == null)
{
throw new Exception("EvaluatorDef must be specified when running LC0 engine");
}
// TODO: do we really want to emulate always here? probably not
// WARNING: above.
bool forceDisableSmartPruning = false;
return(new GameEngineLC0("LZ0_" + suffix, evaluatorDef.Nets[0].Net.NetworkID,
forceDisableSmartPruning, false,
paramsSearch, paramsSelect, evaluatorDef,
null, CeresUserSettingsManager.GetLC0ExecutableFileName()));
}
else
{
return(engineSpec.CreateEngine());
}
}
/// <summary>
/// Constructor.
/// </summary>
/// <param name="id"></param>
/// <param name="nnEvaluator"></param>
/// <param name="searchParams"></param>
/// <param name="childSelectParams"></param>
/// <param name="gameLimitManager"></param>
/// <param name="paramsSearchExecutionModifier"></param>
public GameEngineCeresInProcess(string id, NNEvaluatorDef evaluatorDef,
ParamsSearch searchParams = null,
ParamsSelect childSelectParams = null,
IManagerGameLimit gameLimitManager = null,
ParamsSearchExecutionModifier paramsSearchExecutionModifier = null) : base(id)
{
if (evaluatorDef == null)
{
throw new ArgumentNullException(nameof(evaluatorDef));
}
// Use default settings for search and select params if not specified.
if (searchParams == null)
{
searchParams = new ParamsSearch();
}
if (childSelectParams == null)
{
childSelectParams = new ParamsSelect();
}
// Use default limit manager if not specified.
if (gameLimitManager == null)
{
gameLimitManager = new ManagerGameLimitCeres();
}
ParamsSearchExecutionModifier = paramsSearchExecutionModifier;
EvaluatorDef = evaluatorDef;
SearchParams = searchParams;
GameLimitManager = gameLimitManager;
ChildSelectParams = childSelectParams;
}
/// <summary>
/// Dumps GPU information and runs benchmarks.
/// </summary>
/// <returns>Sum of NPS across all GPUs</returns>
public static (int CountGPU, int SumNPS) DumpGPUBenchmark()
{
Console.WriteLine();
Console.WriteLine("-----------------------------------------------------------------------------------");
Console.WriteLine($"GPU BENCHMARK (benchmark net: {CeresUserSettingsManager.Settings.DefaultNetworkSpecString})");
Console.WriteLine();
Console.WriteLine(NVML.InfoDescriptionHeaderLine1 + " NPS 1 NPS Batch");
Console.WriteLine(NVML.InfoDescriptionHeaderLine2 + " ----- ---------");
int sumNPS = 0;
int countGPU = 0;
foreach (NVMLGPUInfo info in NVML.GetGPUsInfo())
{
NNEvaluatorDef evaluatorDef = NNEvaluatorDef.FromSpecification(CeresUserSettingsManager.Settings.DefaultNetworkSpecString,
"GPU:" + info.ID.ToString());
NNEvaluator evaluator = NNEvaluatorFactory.BuildEvaluator(evaluatorDef);
(float npsSingletons, float npsBigBatch, _) = NNEvaluatorBenchmark.EstNPS(evaluator, false, 512, true, 3);
Console.WriteLine(NVML.GetInfoDescriptionLine(info) + $" {npsSingletons,6:N0} { npsBigBatch,10:N0}");
countGPU++;
sumNPS += (int)npsBigBatch;
evaluator.Dispose();
}
return(countGPU, sumNPS);
}
/// <summary>
/// Standard constructor which builds the evaluator
/// (or two evaluators, if running with overlapping)
/// according to a specified definition.
/// </summary>
/// <param name="evaluatorDef"></param>
public NNEvaluatorSet(NNEvaluatorDef evaluatorDef)
{
if (evaluatorDef == null)
{
throw new ArgumentNullException(nameof(evaluatorDef));
}
EvaluatorDef = evaluatorDef;
}
/// <summary>
/// Constructor.
/// </summary>
public GameEngineDefCeres(string id, NNEvaluatorDef evaluatorDef, ParamsSearch searchParams = null,
ParamsSearchExecutionModifier paramsSearchExecutionPostprocessor = null,
ParamsSelect selectParams = null, IManagerGameLimit overrideTimeManager = null)
: base(id)
{
EvaluatorDef = evaluatorDef;
SearchParams = searchParams ?? new ParamsSearch();
ParamsSearchExecutionPostprocessor = paramsSearchExecutionPostprocessor;
SelectParams = selectParams ?? new ParamsSelect();
OverrideTimeManager = overrideTimeManager;
}
/// <summary>
/// Constructor.
/// </summary>
/// <param name="id"></param>
/// <param name="paramsNN"></param>
/// <param name="uciSetOptionCommands"></param>
/// <param name="callback"></param>
/// <param name="overrideEXE"></param>
public GameEngineDefCeresUCI(string id,
NNEvaluatorDef evaluatorDef,
string[] uciSetOptionCommands = null,
GameEngine.ProgressCallback callback = null,
string overrideEXE = null)
: base(id)
{
EvaluatorDef = evaluatorDef;
UCISetOptionCommands = uciSetOptionCommands;
Callback = callback;
OverrideEXE = overrideEXE;
}
/// <summary>
/// Returns set of backend arguments to configure the
/// backend (based net being used).
/// </summary>
/// <param name="evaluatorDef"></param>
/// <returns></returns>
static string BackendArgumentsString(NNEvaluatorDef evaluatorDef)
{
// LC0 does not genearlly support Int8; map into FP16 silently
NNEvaluatorPrecision precision = evaluatorDef.Nets[0].Net.Precision;
if (precision == NNEvaluatorPrecision.Int8)
{
precision = NNEvaluatorPrecision.FP16;
}
return(LC0EngineArgs.BackendArgumentsString(evaluatorDef.DeviceIndices, precision, evaluatorDef.EqualFractions));
}
/// <summary>
/// Constructor.
/// </summary>
public GameEngineDefCeres(string id, NNEvaluatorDef evaluatorDef, ParamsSearch searchParams = null,
ParamsSearchExecutionModifier paramsSearchExecutionPostprocessor = null,
ParamsSelect selectParams = null, IManagerGameLimit overrideTimeManager = null)
: base(id)
{
// Make a defensive clone of the EvaluatorDef so it will definitely not be shared.
EvaluatorDef = ObjUtils.DeepClone(evaluatorDef);
SearchParams = searchParams ?? new ParamsSearch();
ParamsSearchExecutionPostprocessor = paramsSearchExecutionPostprocessor;
SelectParams = selectParams ?? new ParamsSelect();
OverrideTimeManager = overrideTimeManager;
}
/// <summary>
/// Construtor.
/// </summary>
/// <param name="evaluatorDef"></param>
/// <param name="inStream"></param>
/// <param name="outStream"></param>
/// <param name="searchFinishedEvent"></param>
public UCIManager(NNEvaluatorDef evaluatorDef,
TextReader inStream = null, TextWriter outStream = null,
Action <MCTSManager> searchFinishedEvent = null)
{
InStream = inStream ?? Console.In;
OutStream = outStream ?? Console.Out;
SearchFinishedEvent = searchFinishedEvent;
EvaluatorDef = evaluatorDef;
ParamsSearch = new ParamsSearch();
ParamsSelect = new ParamsSelect();
}
private static void LaunchUCI(string keyValueArgs)
{
FeatureUCIParams uciParams = FeatureUCIParams.ParseUCICommand(keyValueArgs);
NNEvaluatorDef evaluatorDef = new NNEvaluatorDef(uciParams.NetworkSpec.ComboType, uciParams.NetworkSpec.NetDefs,
uciParams.DeviceSpec.ComboType, uciParams.DeviceSpec.Devices);
Console.WriteLine($"Network evaluation configured to use: {evaluatorDef.ToString()}");
UCIManager ux = new UCIManager(evaluatorDef, null, null, disablePruning: uciParams.Pruning == false);
Console.WriteLine();
Console.WriteLine("Entering UCI command processing mode.");
ux.PlayUCI();
}
/// <summary>
/// Returns an LC0Engine object configured according to specified settings.
/// </summary>
/// <param name="paramsSearch"></param>
/// <param name="paramsSelect"></param>
/// <param name="evaluatorDef"></param>
/// <param name="network"></param>
/// <param name="resetStateAndCachesBeforeMoves"></param>
/// <param name="emulateCeresOptions"></param>
/// <param name="verboseOutput"></param>
/// <param name="overrideEXE"></param>
/// <returns></returns>
public static LC0Engine GetLC0Engine(ParamsSearch paramsSearch,
ParamsSelect paramsSelect,
NNEvaluatorDef evaluatorDef,
INNWeightsFileInfo network,
bool resetStateAndCachesBeforeMoves,
bool emulateCeresOptions,
bool verboseOutput,
bool forceDisableSmartPruning,
string overrideEXE = null,
bool alwaysFillHistory = false)
{
(string EXE, string lzOptions) = GetLC0EngineOptions(paramsSearch, paramsSelect, evaluatorDef, network,
emulateCeresOptions, verboseOutput, overrideEXE,
forceDisableSmartPruning, alwaysFillHistory);
return(new LC0Engine(EXE, lzOptions, resetStateAndCachesBeforeMoves));
}
/// <summary>
/// Returns the network and device arguments string matching specified NNEvaluatorDef.
/// </summary>
/// <param name="evaluatorDef"></param>
/// <returns></returns>
static string ExtraArgsForEvaluator(NNEvaluatorDef evaluatorDef)
{
string netSpecString = NNNetSpecificationString.ToSpecificationString(evaluatorDef.NetCombo, evaluatorDef.Nets);
if (!netSpecString.Contains("Network="))
{
throw new Exception("Unsupported network specification");
}
string deviceSpecString = NNDevicesSpecificationString.ToSpecificationString(evaluatorDef.DeviceCombo, evaluatorDef.Devices);
if (!deviceSpecString.Contains("Device="))
{
throw new Exception("Unsupported device specification");
}
return("UCI " + netSpecString + " " + deviceSpecString);
}
/// <summary>
/// Constructor.
/// </summary>
/// <param name="nnEvaluatorDef"></param>
/// <param name="paramsSearch"></param>
/// <param name="paramsSelect"></param>
/// <param name="searchLimit"></param>
public ParamsSearchExecutionChooser(NNEvaluatorDef nnEvaluatorDef,
ParamsSearch paramsSearch,
ParamsSelect paramsSelect,
SearchLimit searchLimit)
{
// Make sure params arguments look initialized
if (nnEvaluatorDef == null)
{
throw new ArgumentNullException(nameof(nnEvaluatorDef));
}
NNEvaluatorDef = nnEvaluatorDef;
ParamsSearch = paramsSearch;
ParamsSelect = paramsSelect;
SearchLimit = searchLimit with {
};
}
/// <summary>
/// Constructor.
/// </summary>
/// <param name="id"></param>
/// <param name="evaluatorDef"></param>
/// <param name="forceDisableSmartPruning"></param>
/// <param name="searchParamsEmulate"></param>
/// <param name="selectParamsEmulate"></param>
/// <param name="overrideEXE"></param>
/// <param name="extraCommandLineArgs"></param>
public GameEngineDefLC0(string id,
NNEvaluatorDef evaluatorDef,
bool forceDisableSmartPruning,
ParamsSearch searchParamsEmulate = null,
ParamsSelect selectParamsEmulate = null,
string overrideEXE = null,
string extraCommandLineArgs = null)
: base(id)
{
if ((SearchParamsEmulate == null) != (SelectParamsEmulate == null))
{
throw new ArgumentException("SearchParamsEmulate and SelectParamsEmulate must be both provided or not");
}
// Verify compatability of evaluator for LC0
if (evaluatorDef == null)
{
throw new ArgumentNullException(nameof(evaluatorDef));
}
if (evaluatorDef.Nets.Length != 1)
{
throw new Exception("Exactly one network must be specified for use with LC0.");
}
if (evaluatorDef.Nets[0].Net.Type != NNEvaluatorType.LC0Library)
{
throw new Exception("Network Type must be LC0Library");
}
if (evaluatorDef.NetCombo != NNEvaluatorNetComboType.Single)
{
throw new Exception("Network Type must be Single");
}
ID = id;
// Make a defensive clone of the EvaluatorDef so it will definitely not be shared.
EvaluatorDef = ObjUtils.DeepClone(evaluatorDef);
ForceDisableSmartPruning = forceDisableSmartPruning;
SearchParamsEmulate = searchParamsEmulate;
SelectParamsEmulate = selectParamsEmulate;
OverrideEXE = overrideEXE;
ExtraCommandLineArgs = extraCommandLineArgs;
}
/// <summary>
/// Construtor.
/// </summary>
/// <param name="evaluatorDef"></param>
/// <param name="inStream"></param>
/// <param name="outStream"></param>
/// <param name="searchFinishedEvent"></param>
public UCIManager(NNEvaluatorDef evaluatorDef,
TextReader inStream = null, TextWriter outStream = null,
Action <MCTSManager> searchFinishedEvent = null,
bool disablePruning = false)
{
InStream = inStream ?? Console.In;
OutStream = outStream ?? Console.Out;
SearchFinishedEvent = searchFinishedEvent;
EvaluatorDef = evaluatorDef;
ParamsSearch = new ParamsSearch();
ParamsSelect = new ParamsSelect();
if (disablePruning)
{
ParamsSearch.FutilityPruningStopSearchEnabled = false;
}
}
/// <summary>
/// Constructor.
/// </summary>
/// <param name="id"></param>
/// <param name="evaluatorDef"></param>
/// <param name="searchParams"></param>
/// <param name="childSelectParams"></param>
/// <param name="gameLimitManager"></param>
/// <param name="paramsSearchExecutionModifier"></param>
/// <param name="logFileName"></param>
public GameEngineCeresInProcess(string id, NNEvaluatorDef evaluatorDef,
ParamsSearch searchParams = null,
ParamsSelect childSelectParams = null,
IManagerGameLimit gameLimitManager = null,
ParamsSearchExecutionModifier paramsSearchExecutionModifier = null,
string logFileName = null) : base(id)
{
if (evaluatorDef == null)
{
throw new ArgumentNullException(nameof(evaluatorDef));
}
// Use default settings for search and select params if not specified.
if (searchParams == null)
{
searchParams = new ParamsSearch();
}
if (childSelectParams == null)
{
childSelectParams = new ParamsSelect();
}
// Use default limit manager if not specified.
if (gameLimitManager == null)
{
gameLimitManager = new ManagerGameLimitCeres();
}
ParamsSearchExecutionModifier = paramsSearchExecutionModifier;
EvaluatorDef = evaluatorDef;
SearchParams = searchParams;
GameLimitManager = gameLimitManager;
ChildSelectParams = childSelectParams;
SearchLogFileName = logFileName;
VerboseMoveStats = CeresUserSettingsManager.Settings.VerboseMoveStats;
if (!string.IsNullOrEmpty(CeresUserSettingsManager.Settings.SearchLogFile))
{
SearchLogFileName = CeresUserSettingsManager.Settings.SearchLogFile;
}
}
/// <summary>
/// Constructor for a NN evaluator (either local or remote) with specified parameters.
/// </summary>
/// <param name="paramsNN"></param>
/// <param name="saveToCache"></param>
/// <param name="instanceID"></param>
/// <param name="lowPriority"></param>
public LeafEvaluatorNN(NNEvaluatorDef evaluatorDef, NNEvaluator evaluator,
bool saveToCache,
bool lowPriority,
PositionEvalCache cache,
Func <MCTSIterator, int> batchEvaluatorIndexDynamicSelector)
{
rawPosArray = posArrayPool.Rent(NNEvaluatorDef.MAX_BATCH_SIZE);
EvaluatorDef = evaluatorDef;
SaveToCache = saveToCache;
LowPriority = lowPriority;
Cache = cache;
this.BatchEvaluatorIndexDynamicSelector = batchEvaluatorIndexDynamicSelector;
Batch = new EncodedPositionBatchFlat(EncodedPositionType.PositionOnly, NNEvaluatorDef.MAX_BATCH_SIZE);
if (evaluatorDef.Location == NNEvaluatorDef.LocationType.Local)
{
localEvaluator = evaluator;// isEvaluator1 ? Params.Evaluator1 : Params.Evaluator2;
}
else
{
throw new NotImplementedException();
}
// TODO: auto-estimate performance
#if SOMEDAY
for (int i = 0; i < 10; i++)
{
// using (new TimingBlock("benchmark"))
{
float[] splits = WFEvalNetBenchmark.GetBigBatchNPSFractions(((WFEvalNetCompound)localEvaluator).Evaluators);
Console.WriteLine(splits[0] + " " + splits[1] + " " + splits[2] + " " + splits[3]);
(float estNPSSingletons, float estNPSBigBatch) = WFEvalNetBenchmark.EstNPS(localEvaluator);
Console.WriteLine(estNPSSingletons + " " + estNPSBigBatch);
}
}
#endif
}
/// <summary>
/// Constructor.
/// </summary>
/// <param name="id"></param>
/// <param name="evaluatorDef"></param>
/// <param name="forceDisableSmartPruning"></param>
/// <param name="searchParamsEmulate"></param>
/// <param name="selectParamsEmulate"></param>
/// <param name="overrideEXE"></param>
public GameEngineDefLC0(string id,
NNEvaluatorDef evaluatorDef,
bool forceDisableSmartPruning,
ParamsSearch searchParamsEmulate = null,
ParamsSelect selectParamsEmulate = null,
string overrideEXE = null)
: base(id)
{
if ((SearchParamsEmulate == null) != (SelectParamsEmulate == null))
{
throw new ArgumentException("SearchParamsEmulate and SelectParamsEmulate must be both provided or not");
}
// Verify compatability of evaluator for LC0
if (evaluatorDef == null)
{
throw new ArgumentNullException(nameof(evaluatorDef));
}
if (evaluatorDef.Nets.Length != 1)
{
throw new Exception("Exactly one network must be specified for use with LC0.");
}
if (evaluatorDef.Nets[0].Net.Type != NNEvaluatorType.LC0Library)
{
throw new Exception("Network Type must be LC0Library");
}
if (evaluatorDef.NetCombo != NNEvaluatorNetComboType.Single)
{
throw new Exception("Network Type must be Single");
}
ID = id;
EvaluatorDef = evaluatorDef ?? throw new ArgumentNullException(nameof(evaluatorDef));
ForceDisableSmartPruning = forceDisableSmartPruning;
SearchParamsEmulate = searchParamsEmulate;
SelectParamsEmulate = selectParamsEmulate;
OverrideEXE = overrideEXE;
}
/// <summary>
/// Constructor
/// </summary>
/// <param name="searchParams"></param>
/// <param name="selectParams"></param>
/// <param name="paramsNN"></param>
/// <param name="id"></param>
/// <param name="networkID"></param>
/// <param name="emulateCeresSettings"></param>
/// <param name="setupAction"></param>
/// <param name="overrideEXE"></param>
public GameEngineLC0(string id, string networkID, bool forceDisableSmartPruning = false,
bool emulateCeresSettings = false,
ParamsSearch searchParams = null, ParamsSelect selectParams = null,
NNEvaluatorDef paramsNN = null,
Action setupAction = null,
string overrideEXE = null,
bool verbose = false,
bool alwaysFillHistory = false) : base(id)
{
SetupAction = setupAction;
if (SetupAction != null)
{
SetupAction();
}
bool resetStateAndCachesBeforeMoves = searchParams != null && !searchParams.TreeReuseEnabled;
LC0Engine = LC0EngineConfigured.GetLC0Engine(searchParams, selectParams, paramsNN,
NNWeightsFiles.LookupNetworkFile(networkID), emulateCeresSettings,
resetStateAndCachesBeforeMoves, verbose,
forceDisableSmartPruning, overrideEXE,
alwaysFillHistory);
}
public static void TestSF(int index, bool gitVersion)
{
NNEvaluatorDef evalDef1 = NNEvaluatorDefFactory.FromSpecification("LC0:j94-100", "GPU:" + index);
GameEngineDefCeres engineDefCeres1 = new GameEngineDefCeres("CeresInProc", evalDef1,
new ParamsSearch(), null, new ParamsSelect(),
null, "CeresSF.log.txt");
SearchLimit limitCeres = SearchLimit.SecondsForAllMoves(60, 1.25f) * 0.15f;
SearchLimit limitSF = limitCeres * 1.5f;
GameEngineDef engineDefCeresUCIGit = new GameEngineDefCeresUCI("CeresUCIGit", evalDef1, overrideEXE: @"C:\ceres\releases\v0.88\ceres.exe");
EnginePlayerDef playerCeres = new EnginePlayerDef(gitVersion ? engineDefCeresUCIGit : engineDefCeres1,
limitCeres);
EnginePlayerDef playerSF = new EnginePlayerDef(engineDefStockfish13, limitSF);
TournamentDef def = new TournamentDef("TOURN", playerCeres, playerSF);
def.OpeningsFileName = "TCEC1819.pgn";
//def.NumGamePairs = 10;
def.ShowGameMoves = false;
TournamentManager runner = new TournamentManager(def, 1);
TournamentResultStats results;
TimingStats stats = new TimingStats();
using (new TimingBlock(stats, TimingBlock.LoggingType.None))
{
results = runner.RunTournament();
}
Console.WriteLine();
Console.WriteLine($"Tournament completed in {stats.ElapsedTimeSecs,8:F2} seconds.");
Console.WriteLine(playerCeres + " " + results.GameOutcomesString);
}
/// Returns an NNEvaluator corresponding to speciifed strings with network and device specifications. /// </summary> /// <param name="netSpecificationString"></param> /// <param name="deviceSpecificationString"></param> /// <returns></returns> public static NNEvaluator FromSpecification(string netSpecificationString, string deviceSpecificationString) => NNEvaluatorDef.FromSpecification(netSpecificationString, deviceSpecificationString).ToEvaluator();
/// <summary>
/// Dump two sets of parameters, optionally showing only differences.
/// </summary>
public static void DumpParams(TextWriter writer, bool differentOnly,
GameEngineUCISpec externalEngine1Spec, GameEngineUCISpec externalEngine2Spec,
NNEvaluatorDef evaluatorDef1, NNEvaluatorDef evaluatorDef2,
SearchLimit searchLimit1, SearchLimit searchLimit2,
ParamsSelect selectParams1, ParamsSelect selectParams2,
ParamsSearch searchParams1, ParamsSearch searchParams2,
IManagerGameLimit timeManager1, IManagerGameLimit timeManager2,
ParamsSearchExecution paramsSearchExecution1,
ParamsSearchExecution paramsSearchExecution2)
{
writer.WriteLine("\r\n-----------------------------------------------------------------------");
writer.WriteLine("ENGINE 1 Options Modifications from Default");
if (evaluatorDef1 != null)
{
writer.WriteLine("Ceres Evaluator : " + evaluatorDef1.ToString());
}
else
{
writer.Write("External UCI : " + externalEngine1Spec);
}
writer.Write(ObjUtils.FieldValuesDumpString <SearchLimit>(searchLimit1, SearchLimit.NodesPerMove(1), differentOnly));
// writer.Write(ObjUtils.FieldValuesDumpString<NNEvaluatorDef>(Def.NNEvaluators1.EvaluatorDef, new ParamsNN(), differentOnly));
writer.Write(ObjUtils.FieldValuesDumpString <ParamsSelect>(selectParams1, new ParamsSelect(), differentOnly));
writer.Write(ObjUtils.FieldValuesDumpString <ParamsSearch>(searchParams1, new ParamsSearch(), differentOnly));
DumpTimeManagerDifference(true, null, timeManager1);
writer.Write(ObjUtils.FieldValuesDumpString <ParamsSearchExecution>(paramsSearchExecution1, new ParamsSearchExecution(), differentOnly));
writer.WriteLine("\r\n-----------------------------------------------------------------------");
writer.WriteLine("ENGINE 2 Options Modifications from Engine 1");
bool evaluatorsDifferent = false;
bool eval2TypeDifferent = (evaluatorDef1 == null) != (evaluatorDef2 == null);
if (eval2TypeDifferent)
{
evaluatorsDifferent = true;
}
else
{
if (evaluatorDef1 != null)
{
evaluatorsDifferent = evaluatorDef1.ToString() != evaluatorDef2.ToString();
}
else
{
evaluatorsDifferent = externalEngine1Spec.ToString() != externalEngine2Spec.ToString();
}
}
if (!differentOnly || evaluatorsDifferent)
{
if (evaluatorDef1 != null && evaluatorDef2 != null)
{
writer.WriteLine("Ceres Evaluator : " + evaluatorDef2.ToString());
}
else
{
writer.Write("External UCI : " + externalEngine2Spec);
}
}
if (searchLimit2 != null)
{
writer.Write(ObjUtils.FieldValuesDumpString <SearchLimit>(searchLimit2, searchLimit1, differentOnly));
// writer.Write(ObjUtils.FieldValuesDumpString<NNEvaluatorDef>(Def.NNEvaluators1.EvaluatorDef, Def.NNEvaluators2.EvaluatorDef, differentOnly));
writer.Write(ObjUtils.FieldValuesDumpString <ParamsSelect>(selectParams2, selectParams1, differentOnly));
writer.Write(ObjUtils.FieldValuesDumpString <ParamsSearch>(searchParams2, searchParams1, differentOnly));
DumpTimeManagerDifference(differentOnly, timeManager1, timeManager2);
writer.Write(ObjUtils.FieldValuesDumpString <ParamsSearchExecution>(paramsSearchExecution2, paramsSearchExecution1, differentOnly));
}
}
public static void Analyze(string fenAndMoves, SearchLimit searchLimit,
NNEvaluatorDef evaluatorDef,
bool forceDisablePruning,
LC0Engine lc0Engine = null,
GameEngine comparisonEngine = null,
bool verbose = false)
{
Console.WriteLine("=============================================================================");
Console.WriteLine("Analyzing FEN : " + fenAndMoves);
Console.WriteLine("Search limit : " + searchLimit.ToString());
Console.WriteLine("Ceres evaluator : " + evaluatorDef.ToString());
if (comparisonEngine != null)
{
Console.WriteLine("Opponent : " + comparisonEngine.ToString());
}
Console.WriteLine();
Console.WriteLine();
NNEvaluatorSet nnEvaluators = new NNEvaluatorSet(evaluatorDef);
// Warmup (in parallel)
lc0Engine?.DoSearchPrepare();
Parallel.Invoke(
() => nnEvaluators.Warmup(true),
() => comparisonEngine?.Warmup());
bool ceresDone = false;
lastInfoUpdate = DateTime.Now;
UCISearchInfo lastCeresInfo = null;
// Launch Ceres
MCTSearch ceresResults = null;
Task searchCeres = Task.Run(() =>
{
ParamsSearch searchParams = new ParamsSearch();
searchParams.FutilityPruningStopSearchEnabled = !forceDisablePruning;
PositionWithHistory positionWithHistory = PositionWithHistory.FromFENAndMovesUCI(fenAndMoves);
ceresResults = new MCTSearch();
ceresResults.Search(nnEvaluators, new ParamsSelect(), searchParams, null, null,
null, positionWithHistory, searchLimit, verbose, DateTime.Now, null,
manager => lastCeresInfo = new UCISearchInfo(UCIInfo.UCIInfoString(manager), null, null), false, true);
});
// Possibly launch search for other engine
Task searchComparison = null;
if (lc0Engine != null || comparisonEngine != null)
{
searchComparison = Task.Run(() =>
{
if (lc0Engine != null)
{
lc0Engine.DoSearchPrepare();
lc0Engine.AnalyzePositionFromFENAndMoves(fenAndMoves, searchLimit);
}
else
{
comparisonEngine.Search(PositionWithHistory.FromFENAndMovesUCI(fenAndMoves), searchLimit, verbose: true);
}
});
}
;
while (!searchCeres.IsCompleted || (searchComparison != null && !searchComparison.IsCompleted))
{
Thread.Sleep(1000);
//Console.WriteLine(DateTime.Now + " --> " + lastCeresInfo?.PVString + " OTHER " + comparisonEngine?.UCIInfo?.RawString);
int numCharactersSame = int.MaxValue;
if (lastCeresInfo?.PVString != null || comparisonEngine?.UCIInfo?.RawString != null)
{
if (lastCeresInfo != null && comparisonEngine?.UCIInfo != null)
{
numCharactersSame = 0;
string pv1 = lastCeresInfo.PVString;
UCISearchInfo lastComparisonInfo = comparisonEngine.UCIInfo;
string pv2 = lastComparisonInfo.PVString;
while (pv1.Length > numCharactersSame &&
pv2.Length > numCharactersSame &&
pv1[numCharactersSame] == pv2[numCharactersSame])
{
numCharactersSame++;
}
}
}
if (lastCeresInfo != null)
{
WriteUCI("Ceres", lastCeresInfo, numCharactersSame);
}
if (comparisonEngine != null)
{
WriteUCI(comparisonEngine.ID, comparisonEngine.UCIInfo, numCharactersSame);
}
Console.WriteLine();
}
searchCeres.Wait();
searchComparison?.Wait();
string infoUpdate = UCIInfo.UCIInfoString(ceresResults.Manager);
double q2 = ceresResults.SearchRootNode.Q;
//SearchPrincipalVariation pv2 = new SearchPrincipalVariation(worker2.Root);
MCTSPosTreeNodeDumper.DumpPV(ceresResults.SearchRootNode, true);
}
/// <summary>
/// Returns the executable location and program arguments appropriate
/// for LC0 given specified Ceres parameters
/// (optionally emulating them to the degree possible).
/// </summary>
/// <param name="paramsSearch"></param>
/// <param name="paramsSelect"></param>
/// <param name="evaluatorDef"></param>
/// <param name="network"></param>
/// <param name="emulateCeresOptions"></param>
/// <param name="verboseOutput"></param>
/// <param name="overrideEXE"></param>
/// <param name="forceDisableSmartPruning"></param>
/// <param name="alwaysFillHistory"></param>
/// <returns></returns>
public static (string, string) GetLC0EngineOptions(ParamsSearch paramsSearch,
ParamsSelect paramsSelect,
NNEvaluatorDef evaluatorDef,
INNWeightsFileInfo network,
bool emulateCeresOptions,
bool verboseOutput,
string overrideEXE = null,
bool forceDisableSmartPruning = false,
bool alwaysFillHistory = false)
{
if (paramsSearch == null)
{
paramsSearch = new ParamsSearch();
}
if (paramsSelect == null)
{
paramsSelect = new ParamsSelect();
}
//fail int8 string precisionStr = MCTSParams.PRECISION == WFEvalNetTensorRT.TRTPrecision.Int8 ? "trt-int8" : "cudnn-fp16";
// Must reverse values to conform to LZ0 convention
float FPU_MULTIPLIER = paramsSelect.FPUMode == ParamsSelect.FPUType.Reduction ? -1.0f : 1.0f;
// TODO: plug in both versions of Centiapwn to low level Leela code
string fpuVals = $"--fpu-value={FPU_MULTIPLIER * paramsSelect.FPUValue} --fpu-value-at-root={FPU_MULTIPLIER * paramsSelect.FPUValueAtRoot} ";
string strategyStr = paramsSelect.FPUMode == ParamsSelect.FPUType.Absolute ? "absolute " : "reduction ";
string fpuStr = fpuVals + "--fpu-strategy=" + strategyStr;
string strategyAtRootStr = paramsSelect.FPUModeAtRoot == ParamsSelect.FPUType.Absolute ? "absolute " : "reduction ";
string fpuStrRoot = paramsSelect.FPUModeAtRoot == ParamsSelect.FPUType.Same ? "--fpu-strategy-at-root=same "
: "--fpu-strategy-at-root=" + strategyAtRootStr;
string netSourceFile = network.FileName;
int minibatchSize = 256; // LC0 default
// If GPUs have equal fractions then we use demux where only 2 threads needed,
// otherwise we use roundrobin and best is 1 + number of GPUS
// Note that with increasing threads Leela plays significantly non-deterministically and somewhat worse
int NUM_THREADS = evaluatorDef.EqualFractions ? 2 : evaluatorDef.Devices.Length + 1;
string lzOptions = "--nodes-as-playouts "; // essential to get same behavior as Ceres with go nodes command
lzOptions += "--multi-gather "; // greatly improves search speed
if (forceDisableSmartPruning || (emulateCeresOptions && !paramsSearch.FutilityPruningStopSearchEnabled))
{
lzOptions += " --smart-pruning-factor=0 ";
}
// Default nncache is only 200_000 but big tournaments (TCEC 19) have used as high as 20_000_000.
// To keep memory requires reasonable for typical systems we default to a value in between.
// However note that for very small nets such as 128x10 it may be faster to uze zero nncache.
const int LC0_CACHE_SIZE = 5_000_000;
int MOVE_OVERHEAD = (int)(new ParamsSearch().MoveOverheadSeconds * 1000);
lzOptions += $"--move-overhead={MOVE_OVERHEAD} ";
if (USE_LC0_SMALL_SEARCH_SETTINGS)
{
// Works better for smaller searchs (such as 1000 nodes)
lzOptions += " --max-collision-visits=32 ";
}
else
{
// Much faster for large searches with multigather enabled.
lzOptions += " --max-out-of-order-evals-factor=2.4 --max-collision-events=500 --max-collision-visits=500 ";
}
if (alwaysFillHistory)
{
lzOptions += $" --history-fill=always ";
}
if (emulateCeresOptions)
{
bool useNNCache = evaluatorDef.CacheMode > PositionEvalCache.CacheMode.None ||
paramsSearch.Execution.TranspositionMode > TranspositionMode.None;
int cacheSize = useNNCache ? LC0_CACHE_SIZE : 0;
lzOptions += $@"-w {netSourceFile} -t {NUM_THREADS} --minibatch-size={minibatchSize} " +
// " --policy-softmax-temp=2.2 --backend=cudnn-fp16 ";
$" --policy-softmax-temp={paramsSelect.PolicySoftmax} --cache-history-length={evaluatorDef.NumCacheHashPositions - 1} " +
// $" --score-type=win_percentage" +
BackendArgumentsString(evaluatorDef) +
//"--backend=multiplexing --backend-opts=(backend=cudnn-fp16,gpu=0),(backend=cudnn-fp16,gpu=1),(backend=cudnn-fp16,gpu=2),(backend=cudnn-fp16,gpu=3) " +
// $"--backend={precisionStr} --backend-opts=gpu={SearchParamsNN.GPU_ID_LEELA_UCI} " +
$"{fpuStr} {fpuStrRoot} " +
$" --no-sticky-endgames ";
// + --no-out-of-order-eval"; // *** NOTE: if we add this flag, LZ0 seems to play a little different and better. TODO: study this, should we adopt?
lzOptions += $" --cpuct-factor={paramsSelect.CPUCTFactor} --cpuct-base={paramsSelect.CPUCTBase} --cpuct={paramsSelect.CPUCT} --nncache={cacheSize} ";
// lzOptions += $" --max-collision-visits={paramsSearch.MAX_COLLISIONS + 1 }"; // Must increment by 1 to make comparable (also, LC0 hangs at value ot zero)
}
else
{
// Mostly we let Leela use default options, except to make it fair
// we use a large nncache and number of threads appropriate for the number of GPUs in use
// lzOptions = $@"-w {weightsDir}\{netSourceFile} --minibatch-size={minibatchSize} -t {paramsNN.NNEVAL_NUM_GPUS + 1} " +
lzOptions += $@"-w {netSourceFile} -t {NUM_THREADS} " +
// $"--score-type=win_percentage " +
$"--nncache={LC0_CACHE_SIZE} " +
// like TCEC 10, only 5% benefit $"--max-prefetch=160 --max-collision-events=917 " +
BackendArgumentsString(evaluatorDef);
}
string tbPath = CeresUserSettingsManager.Settings.TablebaseDirectory;
if (paramsSearch.EnableTablebases)
{
lzOptions += (@$ " --syzygy-paths=#{tbPath}# ").Replace("#", "\"");
}
if (verboseOutput)
{
lzOptions += " --verbose-move-stats ";
}
string EXE = CeresUserSettingsManager.GetLC0ExecutableFileName();
#if EXPERIMENTAL
const bool LZ_USE_TRT = false; // NOTE: if true, the GPU is seemingly currently hardcoded to 3. The max batch size is 512
if (LZ_USE_TRT)
{
if (network.NetworkID == "59999")
{
EXE = @"C:\dev\lc0\19May\lc0\build\lc0_59999.exe";
}
else if (network.NetworkID == "42767")
{
EXE = @"C:\dev\lc0\19May\lc0\build\lc0_42767.exe";
}
else
{
throw new Exception("Unknown net for EXE " + network.NetworkID);
}
if (evaluatorDef.Nets[0].Net.Precision == NNEvaluatorPrecision.Int8)
{
lzOptions = lzOptions.Replace("cudnn-fp16", "trt-int8");
}
else if (evaluatorDef.Nets[0].Net.Precision == NNEvaluatorPrecision.FP16)
{
lzOptions = lzOptions.Replace("cudnn-fp16", "trt-fp16");
}
else
{
throw new NotImplementedException();
}
}
#endif
if (overrideEXE != null)
{
EXE = overrideEXE;
}
return(EXE, lzOptions);
}
/// <summary>
/// Test code. Currently configured for 703810 using 2A100 versus LC0.
/// </summary>
public static void Test()
{
string ETHERAL_EXE = @"\\synology\dev\chess\engines\Ethereal12.75-x64-popcnt-avx2.exe";
string SF11_EXE = @"\\synology\dev\chess\engines\stockfish_11_x64_bmi2.exe";
string SF12_EXE = @"\\synology\dev\chess\engines\stockfish_20090216_x64_avx2.exe";
GameEngineUCISpec specEthereal = new GameEngineUCISpec("Ethereal12", ETHERAL_EXE);
GameEngineUCISpec specSF = new GameEngineUCISpec("SF12", SF12_EXE);
GameEngineUCISpec specLC0 = new GameEngineUCISpec("LC0", "lc0.exe");
// 66511
//NNEvaluatorDef def1 = NNEvaluatorDefFactory.SingleNet("j92-280", NNEvaluatorType.LC0Dll,1);
// NNEvaluatorDef def0 = NNEvaluatorDefFactory.FromSpecification("LC0:j92-280", "GPU:1:POOLED");// POOLED");//:POOLED");
// NNEvaluatorDef def1 = NNEvaluatorDefFactory.FromSpecification("LC0:66666", "GPU:1:POOLED");// POOLED");//:POOLED");
NNEvaluatorDef evalDef1 = NNEvaluatorDefFactory.FromSpecification("LC0:703810", "GPU:0,1"); // POOLED");//:POOLED");
NNEvaluatorDef evalDef2 = NNEvaluatorDefFactory.FromSpecification("LC0:703810", "GPU:0,1"); // POOLED");//:POOLED");
// sv5300 j104.0-10000
// def1.MakePersistent();
//def1.PersistentID = "PERSIST";
// NNEvaluatorDef def2 = NNEvaluatorDefFactory.FromSpecification("LC0:66581", "GPU:3");//:POOLED");
// SearchLimit slLC0 = SearchLimit.NodesPerMove(10_000);
// SearchLimit slEthereal = slLC0 * 875;
// SearchLimit slSF = slLC0 * 875;
//specEthereal = specSF;
string[] extraUCI = null;// new string[] {"setoption name Contempt value 5000" };
const int NUM_THREADS = 32;
const int HASH_SIZE_MB = 2048;
string TB_PATH = CeresUserSettingsManager.Settings.DirTablebases;
SearchLimit limit = SearchLimit.NodesPerMove(1_000);
//limit = SearchLimit.SecondsForAllMoves(60);
//limit = SearchLimit.SecondsForAllMoves(60, 1f);
limit = SearchLimit.SecondsForAllMoves(120, 0.5f);
GameEngineDefCeres engineDefCeres1 = new GameEngineDefCeres("Ceres1", evalDef1, new ParamsSearch(), null, new ParamsSelect(), null);
GameEngineDefCeres engineDefCeres2 = new GameEngineDefCeres("Ceres2", evalDef2, new ParamsSearch(), null, new ParamsSelect(), null);
bool forceDisableSmartPruning = limit.IsNodesLimit;
if (forceDisableSmartPruning)
{
engineDefCeres1.SearchParams.FutilityPruningStopSearchEnabled = false;
engineDefCeres2.SearchParams.FutilityPruningStopSearchEnabled = false;
}
GameEngineDef engineDefEthereal = new GameEngineDefUCI("Etheral", new GameEngineUCISpec("Etheral", ETHERAL_EXE, NUM_THREADS, HASH_SIZE_MB, TB_PATH, uciSetOptionCommands: extraUCI));
GameEngineDef engineDefStockfish11 = new GameEngineDefUCI("SF11", new GameEngineUCISpec("SF11", SF11_EXE, NUM_THREADS, HASH_SIZE_MB, TB_PATH, uciSetOptionCommands: extraUCI));
GameEngineDef engineDefStockfish12 = new GameEngineDefUCI("SF12", new GameEngineUCISpec("SF12", SF12_EXE, NUM_THREADS, HASH_SIZE_MB, TB_PATH, uciSetOptionCommands: extraUCI));
//GameEngineDef engineDefCeresUCI = new GameEngineDefUCI("CeresUCI", new GameEngineUCISpec("CeresUCI", @"c:\dev\ceres\artifacts\release\net5.0\ceres.exe"));
GameEngineDef engineDefCeresUCI = new GameEngineDefCeresUCI("CeresUCI", evalDef1, overrideEXE: @"c:\dev\ceres\artifacts\release\net5.0\ceres.exe");
GameEngineDefLC0 engineDefLC1 = new GameEngineDefLC0("LC0_0", evalDef1, forceDisableSmartPruning, null, null);
GameEngineDefLC0 engineDefLC2 = new GameEngineDefLC0("LC0_1", evalDef2, forceDisableSmartPruning, null, null);
EnginePlayerDef playerCeres1UCI = new EnginePlayerDef(engineDefCeresUCI, limit);
EnginePlayerDef playerCeres2UCI = new EnginePlayerDef(engineDefCeresUCI, limit);
EnginePlayerDef playerCeres1 = new EnginePlayerDef(engineDefCeres1, limit);
EnginePlayerDef playerCeres2 = new EnginePlayerDef(engineDefCeres2, limit);
EnginePlayerDef playerEthereal = new EnginePlayerDef(engineDefEthereal, limit);
EnginePlayerDef playerStockfish11 = new EnginePlayerDef(engineDefStockfish11, limit);
EnginePlayerDef playerStockfish12 = new EnginePlayerDef(engineDefStockfish12, limit);
EnginePlayerDef playerLC0 = new EnginePlayerDef(engineDefLC1, limit);
EnginePlayerDef playerLC1 = new EnginePlayerDef(engineDefLC2, limit);
// def.SearchLimitEngine1 = def.SearchLimitEngine2 = SearchLimit.SecondsForAllMoves(15, 0.25f);
// def.SearchLimitEngine2 = def.SearchLimitEngine2 = SearchLimit.SecondsForAllMoves(15, 0.25f);
//(playerCeres1.EngineDef as GameEngineDefCeres).SearchParams.DrawByRepetitionLookbackPlies = 40;
if (false)
{
// ===============================================================================
SuiteTestDef suiteDef = new SuiteTestDef("Suite", @"\\synology\dev\chess\data\epd\endgame2.epd", playerCeres1, playerCeres2, null);
// suiteDef.MaxNumPositions = 100;
SuiteTestRunner suiteRunner = new SuiteTestRunner(suiteDef);
suiteRunner.Run(1, true, false);
return;
// ===============================================================================
}
// engineDefCeres2.SearchParams.TwofoldDrawEnabled = false;
//engineDefCeres1.SearchParams.TreeReuseEnabled = false;
//engineDefCeres2.SearchParams.TreeReuseEnabled = false;
//engineDefCeres1.SearchParams.FutilityPruningStopSearchEnabled= false;
//engineDefCeres2.SearchParams.FutilityPruningStopSearchEnabled= false;
//engineDefLC0.SearchParamsEmulate.FutilityPruningStopSearchEnabled= false;
//TournamentDef def = new TournamentDef("TOURN", playerCeres1, playerLC0);
TournamentDef def = new TournamentDef("TOURN", playerCeres1, playerLC0);
def.NumGamePairs = 50;
// def.ShowGameMoves = false;
// def.OpeningsFileName = @"HERT_2017\Hert500.pgn";
// def.StartingFEN = "1q6/2n4k/1r1p1pp1/RP1P2p1/2Q1P1P1/2N4P/3K4/8 b - - 8 71";
// def.OpeningsFileName = @"\\synology\dev\chess\data\openings\Drawkiller_500pos_reordered.pgn";//
def.OpeningsFileName = "TCEC19_NoomenSelect.pgn";
// def.OpeningsFileName = "TCEC1819.pgn";
// def.AdjudicationThresholdCentipawns = 500;
// def.AdjudicationThresholdNumMoves = 3;
const int CONCURRENCY = 1;// 15;
TournamentManager runner = new TournamentManager(def, CONCURRENCY);
TournamentResultStats results;
//UCIEngineProcess.VERBOSE = true;
TimingStats stats = new TimingStats();
using (new TimingBlock(stats, TimingBlock.LoggingType.None))
{
results = runner.RunTournament();
}
Console.WriteLine();
Console.WriteLine($"Tournament completed in {stats.ElapsedTimeSecs,8:F2} seconds.");
Console.WriteLine(results.GameOutcomesString);
}
