/// <summary>
/// Attempts to locate and return information about the NN file containing a network with a specified ID.
/// </summary>
/// <param name="netWeightsID"></param>
/// <returns></returns>
public static INNWeightsFileInfo LookupNetworkFile(string netWeightsID, bool throwExceptionIfMissing = true)
{
// First look in set of explicitly registered files
if (RegisteredFiles.TryGetValue(netWeightsID, out INNWeightsFileInfo weightsFile))
return weightsFile;
else
{
// Next check each file in each of the directories registered
foreach (NNWeightsFileDirectory directory in RegisteredDirectories)
{
foreach (string fileName in Directory.EnumerateFiles(directory.DirectoryName, directory.SearchPattern))
{
// Check if the registration source accepts this file and creates an INNWeightsFile
INNWeightsFileInfo file = directory.FilenameToNetworkIDFunc(netWeightsID, fileName);
if (file != null)
{
RegisteredFiles[netWeightsID] = file;
return file;
}
}
}
}
if (throwExceptionIfMissing)
throw new Exception($"Network {netWeightsID} not registered via Register or discoverable via directories specified via NNWeightsFilesLC0.RegisterDirectory method.");
else
return null;
}
/// <summary>
/// Returns an LC0Engine object configured according to specified settings.
/// </summary>
/// <param name="paramsSearch"></param>
/// <param name="paramsSelect"></param>
/// <param name="evaluatorDef"></param>
/// <param name="network"></param>
/// <param name="resetStateAndCachesBeforeMoves"></param>
/// <param name="emulateCeresOptions"></param>
/// <param name="verboseOutput"></param>
/// <param name="overrideEXE"></param>
/// <returns></returns>
public static LC0Engine GetLC0Engine(ParamsSearch paramsSearch,
ParamsSelect paramsSelect,
NNEvaluatorDef evaluatorDef,
INNWeightsFileInfo network,
bool resetStateAndCachesBeforeMoves,
bool emulateCeresOptions,
bool verboseOutput,
bool forceDisableSmartPruning,
string overrideEXE = null,
bool alwaysFillHistory = false)
{
(string EXE, string lzOptions) = GetLC0EngineOptions(paramsSearch, paramsSelect, evaluatorDef, network,
emulateCeresOptions, verboseOutput, overrideEXE,
forceDisableSmartPruning, alwaysFillHistory);
return(new LC0Engine(EXE, lzOptions, resetStateAndCachesBeforeMoves));
}
/// <summary>
///
/// </summary>
/// <param name="networkID"></param>
/// <returns></returns>
public static NNWeightsFileLC0 LookupOrDownload(string networkID)
{
// Try to load file from local disk, return if found.
INNWeightsFileInfo existingFile = NNWeightsFiles.LookupNetworkFile(networkID, false);
if (existingFile != null)
{
return(existingFile as NNWeightsFileLC0);
}
// Download the file.
string baseURL = CeresUserSettingsManager.URLLC0NetworksValidated;
NNWeightsFileLC0Downloader downloader = new NNWeightsFileLC0Downloader(baseURL,
CeresUserSettingsManager.Settings.DirLC0Networks);
string fn = downloader.Download(networkID);
// Load and return the file.
return(new NNWeightsFileLC0(networkID, fn));
}
private void ShowWeightsFileInfo()
{
UCIWriteLine();
if (EvaluatorDef.Nets.Length == 1)
{
INNWeightsFileInfo net = NNWeightsFiles.LookupNetworkFile(EvaluatorDef.Nets[0].Net.NetworkID, false);
string infoStr = net == null ? "(unknown)" : net.ShortStr;
UCIWriteLine($"Loaded network weights: { infoStr}");
}
else
{
Console.WriteLine();
for (int i = 0; i < EvaluatorDef.Nets.Length; i++)
{
INNWeightsFileInfo net = NNWeightsFiles.LookupNetworkFile(EvaluatorDef.Nets[i].Net.NetworkID);
string infoStr = net == null ? "(unknown)" : net.ShortStr;
UCIWriteLine($"Loaded network weights: {i} { infoStr}");
}
}
UCIWriteLine();
}
// TODO: Add a Dispose/Release which calls Dispose on associated LC0DllNNEvaluator
public NNEvaluatorLC0(INNWeightsFileInfo net, int[] gpuIDs, NNEvaluatorPrecision precision = NNEvaluatorPrecision.FP16)
{
if (gpuIDs.Length != 1)
{
throw new ArgumentException(nameof(gpuIDs), "Implementation limitation: one GPU id must be specified");
}
if (precision != NNEvaluatorPrecision.FP16)
{
throw new ArgumentException(nameof(precision), "Implementation: only FP16 supported");
}
isWDL = net.IsWDL;
hasM = net.HasMovesLeft;
policies = new CompressedPolicyVector[MAX_BATCH_SIZE];
w = new FP16[MAX_BATCH_SIZE];
l = isWDL ? new FP16[MAX_BATCH_SIZE] : null;
m = isWDL ? new FP16[MAX_BATCH_SIZE] : null;
// Create NN evaluator and attach to it
//TODO: set precision
//LC0Engine engine = LaunchLC0Server.LaunchProcess(net.LC0WeightsFilename, gpuIDs, precision);
Evaluator = new LC0LibraryNNEvaluator(net.FileName, gpuIDs[0]);
}
/// <summary>
/// Runs the UCI loop.
/// </summary>
public void PlayUCI()
{
// Default to the startpos.
curPositionAndMoves = PositionWithHistory.FromFENAndMovesUCI(Position.StartPosition.FEN, null);
curManager = null;
gameMoveHistory = new List <GameMoveStat>();
while (true)
{
string command = InStream.ReadLine();
switch (command)
{
case null:
case "":
break;
case "uci":
Send("id name Ceres"); // TODO: Add executable version
Send("id author David Elliott and the Ceres Authors");
// todo output options such as:
// option name Logfile type check default false
Send("uciok");
break;
case "setoption":
OutStream.WriteLine("Not processing option " + command);
return;
case "stop":
if (taskSearchCurrentlyExecuting != null && !stopIsPending)
{
stopIsPending = true;
// TODO: cleanup
// Possible race condition, curManager is only set in search callback which may not have hit yet
// Fix eventually by rewriting SerchManager to have a constructor and then non-static Search method,
// os we can get the context in this class directly after construction
while (curManager == null)
{
Thread.Sleep(1); // **** TEMPORARY ***
}
curManager.ExternalStopRequested = true;
if (taskSearchCurrentlyExecuting != null)
{
taskSearchCurrentlyExecuting.Wait();
if (!debug && taskSearchCurrentlyExecuting != null)
{
taskSearchCurrentlyExecuting.Result?.Manager?.Dispose();
}
taskSearchCurrentlyExecuting = null;
}
}
curManager = null;
stopIsPending = false;
break;
case "ponderhit":
throw new NotImplementedException("Ceres does not yet support UCI ponder mode.");
return;
case "xboard":
// ignore
break;
case "debug on":
debug = true;
break;
case "debug off":
debug = false;
break;
case "isready":
InitializeEngineIfNeeded();
Send("readyok");
break;
case "ucinewgame":
gameMoveHistory = new List <GameMoveStat>();
CeresEngine?.ResetGame();
break;
case "quit":
if (curManager != null)
{
curManager.ExternalStopRequested = true;
taskSearchCurrentlyExecuting?.Wait();
}
if (CeresEngine != null)
{
CeresEngine.Dispose();
}
System.Environment.Exit(0);
break;
case string c when c.StartsWith("go"):
if (taskSearchCurrentlyExecuting != null)
{
throw new Exception("Received go command when another search was running and not stopped first");
}
InitializeEngineIfNeeded();
taskSearchCurrentlyExecuting = ProcessGo(command);
break;
case string c when c.StartsWith("position"):
try
{
ProcessPosition(c);
}
catch (Exception e)
{
Send($"Illegal position command: \"{c}\"" + System.Environment.NewLine + e.ToString());
}
break;
// Proprietary commands
case "lc0-config":
if (curManager != null)
{
string netID = EvaluatorDef.Nets[0].Net.NetworkID;
INNWeightsFileInfo netDef = NNWeightsFiles.LookupNetworkFile(netID);
(string exe, string options) = LC0EngineConfigured.GetLC0EngineOptions(null, null, curContext.EvaluatorDef, netDef, false, false);
Console.WriteLine("info string " + exe + " " + options);
}
else
{
Console.WriteLine("info string No search manager created");
}
break;
case "dump-params":
if (curManager != null)
{
curManager.DumpParams();
}
else
{
Console.WriteLine("info string No search manager created");
}
break;
case "dump-processor":
HardwareManager.DumpProcessorInfo();
break;
case "dump-time":
if (curManager != null)
{
curManager.DumpTimeInfo();
}
else
{
Console.WriteLine("info string No search manager created");
}
break;
case "dump-store":
if (curManager != null)
{
using (new SearchContextExecutionBlock(curContext))
curManager.Context.Tree.Store.Dump(true);
}
else
{
Console.WriteLine("info string No search manager created");
}
break;
case "dump-move-stats":
if (curManager != null)
{
using (new SearchContextExecutionBlock(curContext))
curManager.Context.Root.Dump(1, 1, prefixString: "info string ");
}
else
{
Console.WriteLine("info string No search manager created");
}
break;
case "dump-pv":
DumpPV(false);
break;
case "dump-pv-detail":
DumpPV(true);
break;
case "dump-nvidia":
NVML.DumpInfo();
break;
case "waitdone": // proprietary verb
taskSearchCurrentlyExecuting?.Wait();
break;
default:
Console.WriteLine($"error Unknown command: {command}");
break;
}
}
}
/// <summary> /// Registers an individual NN file. /// </summary> /// <param name="netWeightsID"></param> /// <param name="weightsFile"></param> public static void RegisterFile(string netWeightsID, INNWeightsFileInfo weightsFile) => RegisteredFiles[netWeightsID] = weightsFile;
/// <summary>
/// Returns the executable location and program arguments appropriate
/// for LC0 given specified Ceres parameters
/// (optionally emulating them to the degree possible).
/// </summary>
/// <param name="paramsSearch"></param>
/// <param name="paramsSelect"></param>
/// <param name="evaluatorDef"></param>
/// <param name="network"></param>
/// <param name="emulateCeresOptions"></param>
/// <param name="verboseOutput"></param>
/// <param name="overrideEXE"></param>
/// <param name="forceDisableSmartPruning"></param>
/// <param name="alwaysFillHistory"></param>
/// <returns></returns>
public static (string, string) GetLC0EngineOptions(ParamsSearch paramsSearch,
ParamsSelect paramsSelect,
NNEvaluatorDef evaluatorDef,
INNWeightsFileInfo network,
bool emulateCeresOptions,
bool verboseOutput,
string overrideEXE = null,
bool forceDisableSmartPruning = false,
bool alwaysFillHistory = false)
{
if (paramsSearch == null)
{
paramsSearch = new ParamsSearch();
}
if (paramsSelect == null)
{
paramsSelect = new ParamsSelect();
}
//fail int8 string precisionStr = MCTSParams.PRECISION == WFEvalNetTensorRT.TRTPrecision.Int8 ? "trt-int8" : "cudnn-fp16";
// Must reverse values to conform to LZ0 convention
float FPU_MULTIPLIER = paramsSelect.FPUMode == ParamsSelect.FPUType.Reduction ? -1.0f : 1.0f;
// TODO: plug in both versions of Centiapwn to low level Leela code
string fpuVals = $"--fpu-value={FPU_MULTIPLIER * paramsSelect.FPUValue} --fpu-value-at-root={FPU_MULTIPLIER * paramsSelect.FPUValueAtRoot} ";
string strategyStr = paramsSelect.FPUMode == ParamsSelect.FPUType.Absolute ? "absolute " : "reduction ";
string fpuStr = fpuVals + "--fpu-strategy=" + strategyStr;
string strategyAtRootStr = paramsSelect.FPUModeAtRoot == ParamsSelect.FPUType.Absolute ? "absolute " : "reduction ";
string fpuStrRoot = paramsSelect.FPUModeAtRoot == ParamsSelect.FPUType.Same ? "--fpu-strategy-at-root=same "
: "--fpu-strategy-at-root=" + strategyAtRootStr;
string netSourceFile = network.FileName;
int minibatchSize = 256; // LC0 default
// If GPUs have equal fractions then we use demux where only 2 threads needed,
// otherwise we use roundrobin and best is 1 + number of GPUS
// Note that with increasing threads Leela plays significantly non-deterministically and somewhat worse
int NUM_THREADS = evaluatorDef.EqualFractions ? 2 : evaluatorDef.Devices.Length + 1;
string lzOptions = "--nodes-as-playouts "; // essential to get same behavior as Ceres with go nodes command
lzOptions += "--multi-gather "; // greatly improves search speed
if (forceDisableSmartPruning || (emulateCeresOptions && !paramsSearch.FutilityPruningStopSearchEnabled))
{
lzOptions += " --smart-pruning-factor=0 ";
}
// Default nncache is only 200_000 but big tournaments (TCEC 19) have used as high as 20_000_000.
// To keep memory requires reasonable for typical systems we default to a value in between.
// However note that for very small nets such as 128x10 it may be faster to uze zero nncache.
const int LC0_CACHE_SIZE = 5_000_000;
int MOVE_OVERHEAD = (int)(new ParamsSearch().MoveOverheadSeconds * 1000);
lzOptions += $"--move-overhead={MOVE_OVERHEAD} ";
if (USE_LC0_SMALL_SEARCH_SETTINGS)
{
// Works better for smaller searchs (such as 1000 nodes)
lzOptions += " --max-collision-visits=32 ";
}
else
{
// Much faster for large searches with multigather enabled.
lzOptions += " --max-out-of-order-evals-factor=2.4 --max-collision-events=500 --max-collision-visits=500 ";
}
if (alwaysFillHistory)
{
lzOptions += $" --history-fill=always ";
}
if (emulateCeresOptions)
{
bool useNNCache = evaluatorDef.CacheMode > PositionEvalCache.CacheMode.None ||
paramsSearch.Execution.TranspositionMode > TranspositionMode.None;
int cacheSize = useNNCache ? LC0_CACHE_SIZE : 0;
lzOptions += $@"-w {netSourceFile} -t {NUM_THREADS} --minibatch-size={minibatchSize} " +
// " --policy-softmax-temp=2.2 --backend=cudnn-fp16 ";
$" --policy-softmax-temp={paramsSelect.PolicySoftmax} --cache-history-length={evaluatorDef.NumCacheHashPositions - 1} " +
// $" --score-type=win_percentage" +
BackendArgumentsString(evaluatorDef) +
//"--backend=multiplexing --backend-opts=(backend=cudnn-fp16,gpu=0),(backend=cudnn-fp16,gpu=1),(backend=cudnn-fp16,gpu=2),(backend=cudnn-fp16,gpu=3) " +
// $"--backend={precisionStr} --backend-opts=gpu={SearchParamsNN.GPU_ID_LEELA_UCI} " +
$"{fpuStr} {fpuStrRoot} " +
$" --no-sticky-endgames ";
// + --no-out-of-order-eval"; // *** NOTE: if we add this flag, LZ0 seems to play a little different and better. TODO: study this, should we adopt?
lzOptions += $" --cpuct-factor={paramsSelect.CPUCTFactor} --cpuct-base={paramsSelect.CPUCTBase} --cpuct={paramsSelect.CPUCT} --nncache={cacheSize} ";
// lzOptions += $" --max-collision-visits={paramsSearch.MAX_COLLISIONS + 1 }"; // Must increment by 1 to make comparable (also, LC0 hangs at value ot zero)
}
else
{
// Mostly we let Leela use default options, except to make it fair
// we use a large nncache and number of threads appropriate for the number of GPUs in use
// lzOptions = $@"-w {weightsDir}\{netSourceFile} --minibatch-size={minibatchSize} -t {paramsNN.NNEVAL_NUM_GPUS + 1} " +
lzOptions += $@"-w {netSourceFile} -t {NUM_THREADS} " +
// $"--score-type=win_percentage " +
$"--nncache={LC0_CACHE_SIZE} " +
// like TCEC 10, only 5% benefit $"--max-prefetch=160 --max-collision-events=917 " +
BackendArgumentsString(evaluatorDef);
}
string tbPath = CeresUserSettingsManager.Settings.TablebaseDirectory;
if (paramsSearch.EnableTablebases)
{
lzOptions += (@$ " --syzygy-paths=#{tbPath}# ").Replace("#", "\"");
}
if (verboseOutput)
{
lzOptions += " --verbose-move-stats ";
}
string EXE = CeresUserSettingsManager.GetLC0ExecutableFileName();
#if EXPERIMENTAL
const bool LZ_USE_TRT = false; // NOTE: if true, the GPU is seemingly currently hardcoded to 3. The max batch size is 512
if (LZ_USE_TRT)
{
if (network.NetworkID == "59999")
{
EXE = @"C:\dev\lc0\19May\lc0\build\lc0_59999.exe";
}
else if (network.NetworkID == "42767")
{
EXE = @"C:\dev\lc0\19May\lc0\build\lc0_42767.exe";
}
else
{
throw new Exception("Unknown net for EXE " + network.NetworkID);
}
if (evaluatorDef.Nets[0].Net.Precision == NNEvaluatorPrecision.Int8)
{
lzOptions = lzOptions.Replace("cudnn-fp16", "trt-int8");
}
else if (evaluatorDef.Nets[0].Net.Precision == NNEvaluatorPrecision.FP16)
{
lzOptions = lzOptions.Replace("cudnn-fp16", "trt-fp16");
}
else
{
throw new NotImplementedException();
}
}
#endif
if (overrideEXE != null)
{
EXE = overrideEXE;
}
return(EXE, lzOptions);
}
/// <summary>
/// Runs the UCI loop.
/// </summary>
public void PlayUCI()
{
// Default to the startpos.
curPositionAndMoves = PositionWithHistory.FromFENAndMovesUCI(Position.StartPosition.FEN);
gameMoveHistory = new List <GameMoveStat>();
while (true)
{
string command = InStream.ReadLine();
if (uciLogWriter != null)
{
LogWriteLine("IN:", command);
}
switch (command)
{
case null:
case "":
break;
case "uci":
UCIWriteLine($"id name Ceres {CeresVersion.VersionString}");
UCIWriteLine("id author David Elliott and the Ceres Authors");
UCIWriteLine(SetOptionUCIDescriptions);
UCIWriteLine("uciok");
break;
case string c when c.StartsWith("setoption"):
ProcessSetOption(command);
break;
case "stop":
if (taskSearchCurrentlyExecuting != null && !stopIsPending)
{
stopIsPending = true;
// Avoid race condition by mkaing sure the search is already created.
while (CeresEngine.Search?.Manager == null)
{
Thread.Sleep(20);
}
CeresEngine.Search.Manager.ExternalStopRequested = true;
if (taskSearchCurrentlyExecuting != null)
{
taskSearchCurrentlyExecuting.Wait();
// if (!debug && taskSearchCurrentlyExecuting != null) taskSearchCurrentlyExecuting.Result?.Search?.Manager?.Dispose();
taskSearchCurrentlyExecuting = null;
}
}
stopIsPending = false;
break;
case "ponderhit":
throw new NotImplementedException("Ceres does not yet support UCI ponder mode.");
return;
case "xboard":
// ignore
break;
case "debug on":
debug = true;
break;
case "debug off":
debug = false;
break;
case "isready":
InitializeEngineIfNeeded();
UCIWriteLine("readyok");
break;
case "ucinewgame":
gameMoveHistory = new List <GameMoveStat>();
CeresEngine?.ResetGame();
break;
case "quit":
if (taskSearchCurrentlyExecuting != null)
{
CeresEngine.Search.Manager.ExternalStopRequested = true;
taskSearchCurrentlyExecuting?.Wait();
}
if (CeresEngine != null)
{
CeresEngine.Dispose();
}
System.Environment.Exit(0);
break;
case string c when c.StartsWith("go"):
// Possibly another search is already executing.
// The UCI specification is unclear about what to do in this situation.
// Some engines seem to enqueue these for later execution (e.g. Stockfish)
// whereas others (e.g. Python chess) report this as an error condition.
// Currently Ceres waits only a short while for any possible pending search
// to finish (e.g. to avoid a race condition if it is in the process of being shutdown)
// and aborts with an error if search is still in progress.
// It is not viable to wait indefinitely, since (among other reasons)
// the engine needs to monitor for stop commands.
const int MAX_MILLISECONDS_WAIT = 500;
taskSearchCurrentlyExecuting?.Wait(MAX_MILLISECONDS_WAIT);
if (taskSearchCurrentlyExecuting != null && !taskSearchCurrentlyExecuting.IsCompleted)
{
throw new Exception("Received go command when another search was running and not stopped first.");
}
InitializeEngineIfNeeded();
taskSearchCurrentlyExecuting = ProcessGo(command);
break;
case string c when c.StartsWith("position"):
try
{
ProcessPosition(c);
}
catch (Exception e)
{
UCIWriteLine($"Illegal position command: \"{c}\"" + System.Environment.NewLine + e.ToString());
}
break;
// Proprietary commands
case "lc0-config":
if (CeresEngine?.Search != null)
{
string netID = EvaluatorDef.Nets[0].Net.NetworkID;
INNWeightsFileInfo netDef = NNWeightsFiles.LookupNetworkFile(netID);
(string exe, string options) = LC0EngineConfigured.GetLC0EngineOptions(null, null, CeresEngine.Search.Manager.Context.EvaluatorDef, netDef, false, false);
UCIWriteLine("info string " + exe + " " + options);
}
else
{
UCIWriteLine("info string No search manager created");
}
break;
case "dump-params":
if (CeresEngine?.Search != null)
{
CeresEngine?.Search.Manager.DumpParams();
}
else
{
UCIWriteLine("info string No search manager created");
}
break;
case "dump-processor":
HardwareManager.DumpProcessorInfo();
break;
case "dump-time":
if (CeresEngine?.Search != null)
{
CeresEngine?.Search.Manager.DumpTimeInfo(OutStream);
}
else
{
UCIWriteLine("info string No search manager created");
}
break;
case "dump-store":
if (CeresEngine?.Search != null)
{
using (new SearchContextExecutionBlock(CeresEngine.Search.Manager.Context))
CeresEngine.Search.Manager.Context.Tree.Store.Dump(true);
}
else
{
UCIWriteLine("info string No search manager created");
}
break;
case "dump-move-stats":
if (CeresEngine?.Search != null)
{
OutputVerboseMoveStats(CeresEngine.Search.SearchRootNode);
}
else
{
UCIWriteLine("info string No search manager created");
}
break;
case "dump-pv":
DumpPV(false);
break;
case "dump-pv-detail":
DumpPV(true);
break;
case "dump-nvidia":
NVML.DumpInfo();
break;
case "show-tree-plot":
if (CeresEngine?.Search != null)
{
using (new SearchContextExecutionBlock(CeresEngine.Search.Manager.Context))
{
TreePlot.Show(CeresEngine.Search.Manager.Context.Root.Ref);
}
}
else
{
UCIWriteLine("info string No search manager created");
}
break;
case string c when c.StartsWith("save-tree-plot"):
if (CeresEngine?.Search != null)
{
string[] parts = command.Split(" ");
if (parts.Length == 2)
{
string fileName = parts[1];
using (new SearchContextExecutionBlock(CeresEngine.Search.Manager.Context))
{
TreePlot.Save(CeresEngine.Search.Manager.Context.Root.Ref, fileName);
}
}
else if (parts.Length == 1)
{
UCIWriteLine("Filename was not provided");
}
else
{
UCIWriteLine("Filename cannot contain spaces");
}
}
else
{
UCIWriteLine("info string No search manager created");
}
break;
case "waitdone": // proprietary verb used for test driver
taskSearchCurrentlyExecuting?.Wait();
break;
default:
UCIWriteLine($"error Unknown command: {command}");
break;
}
}
}
/// <summary>
/// Constructor when only one GPU is requested.
/// </summary>
/// <param name="net"></param>
/// <param name="gpuID"></param>
/// <param name="precision"></param>
public NNEvaluatorLC0(INNWeightsFileInfo net, int gpuID = 0, NNEvaluatorPrecision precision = NNEvaluatorPrecision.FP16)
: this(net, new int[] { gpuID }, precision)
{
}
