private static void TrainMultiDevice(Symbol symbol, IList <Context> ctx, IList <string> argNames, IList <string> paramNames, IList <string> auxNames, Dictionary <string, NdArray> argParams, Dictionary <string, NdArray> auxParams, int beginEpoch, int endEpoch, int?epochSize, Optimizer optimizer, IDataIter trainData, IDataIter evalData, EvalMetric evalMetric, IList <EpochEndDelegate> epochEndCallback, IList <BatchEndDelegate> batchEndCallback, KvStore kvstore, bool updateOnKvstore, ILog logger, IList <int> workLoadList, Monitor monitor, IList <BatchEndDelegate> evalBatchEndCallback, SymbolGenerate symGen) { if (logger == null) { logger = LogManager.GetLogger(""); } var executorManager = new DataParallelExecutorManager(symbol: symbol, symGen: symGen, ctx: ctx, trainData: trainData, paramNames: paramNames, argNames: argNames, auxNames: auxNames, workLoadList: workLoadList, logger: logger); if (monitor != null) { executorManager.InstallMonitor(monitor); } executorManager.SetParams(argParams, auxParams); Action <int, NdArray, NdArray> updater = null; if (!updateOnKvstore) { updater = Optimizer.GetUpdater(optimizer); } if (kvstore != null) { InitializeKvstore(kvstore: kvstore, paramArrays: executorManager.ParamArrays, argParams: argParams, paramNames: executorManager.ParamNames, updateOnKvstore: updateOnKvstore); } if (updateOnKvstore) { kvstore?.SetOptimizer(optimizer); } //Now start training for (int epoch = 0; epoch < endEpoch - beginEpoch; epoch++) { // Training phase Stopwatch toc = new Stopwatch(); toc.Start(); evalMetric.Reset(); var nbatch = 0; // Iterate over training data. while (true) { var doReset = true; foreach (var dataBatch in trainData) { executorManager.LoadDataBatch(dataBatch); monitor?.Tic(); executorManager.Forward(isTrain: true); executorManager.Backward(); if (updateOnKvstore) { UpdateParamsOnKvstore( executorManager.ParamArrays, executorManager.GradArrays, kvstore); } else { UpdateParams(executorManager.ParamArrays, executorManager.GradArrays, updater: updater, numDevice: ctx.Count, kvstore: kvstore); } monitor?.TocPrint(); // evaluate at end, so we can lazy copy executorManager.UpdateMetric(evalMetric, dataBatch.Label); nbatch += 1; //batch callback (for print purpose) if (batchEndCallback != null) { var batchEndParams = new BatchEndParam(epoch: epoch, nbatch: nbatch, evalMetric: evalMetric, locals: Thread.CurrentThread.CurrentCulture); foreach (var call in batchEndCallback) { call(batchEndParams); } } if (epochSize != null && nbatch >= epochSize) { doReset = false; break; } } if (doReset) { logger.Info($"Epoch[{epoch}] Resetting Data Iterator"); trainData.Reset(); } if (epochSize == null || nbatch >= epochSize) { break; } } logger.Info($"Epoch[{epoch}] Time cost={(toc.ElapsedMilliseconds/1000):.000}"); if (epochEndCallback != null || epoch + 1 == endEpoch) { executorManager.copy_to(argParams, auxParams); } if (epochEndCallback != null) { EpochEndParam epochEndParam = new EpochEndParam(epoch, symbol, argParams, auxParams); foreach (var callitem in epochEndCallback) { callitem(epochEndParam); } } // evaluation if (evalData != null) { evalMetric.Reset(); evalData.Reset(); int i = 0; foreach (var eval_batch in evalData) { executorManager.LoadDataBatch(eval_batch); executorManager.Forward(isTrain: false); executorManager.UpdateMetric(evalMetric, eval_batch.Label); if (evalBatchEndCallback != null) { var batchEndParams = new BatchEndParam(epoch: epoch, nbatch: i, evalMetric: evalMetric, locals: Thread.CurrentThread.CurrentCulture); foreach (var call in evalBatchEndCallback) { call(batchEndParams); } } i++; } var nameValue = evalMetric.get_name_value(); foreach (var item in nameValue) { logger.Info($"Epoch[{epoch}] Validation-{item.Name}={item.Value:0.000}"); } evalData.Reset(); } } }
private static void _train_multi_device(Symbol symbol, List <Context> ctx, List <string> arg_names, List <string> param_names, List <string> aux_names, Dictionary <string, NDArray> arg_params, Dictionary <string, NDArray> aux_params, int begin_epoch, int end_epoch, int?epoch_size, Optimizer optimizer, IDataIter train_data, IDataIter eval_data, EvalMetric eval_metric, List <Action> epoch_end_callback, List <Action <BatchEndParam> > batch_end_callback, KVStore kvstore, bool update_on_kvstore, ILog logger, List <int> work_load_list, Monitor monitor, Action eval_batch_end_callback, SymbolGenerate sym_gen) { if (logger == null) { logger = LogManager.GetLogger(""); } var executor_manager = new DataParallelExecutorManager(symbol: symbol, sym_gen: sym_gen, ctx: ctx, train_data: train_data, param_names: param_names, arg_names: arg_names, aux_names: aux_names, work_load_list: work_load_list, logger: logger); if (monitor != null) { executor_manager.install_monitor(monitor); } executor_manager.set_params(arg_params, aux_params); Action <int, NDArray, NDArray> updater = null; if (!update_on_kvstore) { updater = Optimizer.get_updater(optimizer); } if (kvstore != null) { _initialize_kvstore(kvstore: kvstore, param_arrays: executor_manager.param_arrays, arg_params: arg_params, param_names: executor_manager.param_names, update_on_kvstore: update_on_kvstore); } if (update_on_kvstore) { kvstore.set_optimizer(optimizer); } //Now start training for (int epoch = 0; epoch < end_epoch - begin_epoch; epoch++) { // Training phase Stopwatch toc = new Stopwatch(); toc.Start(); eval_metric.Reset(); var nbatch = 0; // Iterate over training data. while (true) { var do_reset = true; foreach (var data_batch in train_data) { executor_manager.load_data_batch(data_batch); monitor?.Tic(); executor_manager.Forward(is_train: true); executor_manager.Backward(); if (update_on_kvstore) { _update_params_on_kvstore( executor_manager.param_arrays, executor_manager.grad_arrays, kvstore); } else { _update_params(executor_manager.param_arrays, executor_manager.grad_arrays, updater: updater, num_device: ctx.Count, kvstore: kvstore); } monitor?.toc_print(); // evaluate at end, so we can lazy copy executor_manager.update_metric(eval_metric, data_batch.label); nbatch += 1; //batch callback (for print purpose) if (batch_end_callback != null) { var batch_end_params = new BatchEndParam(epoch: epoch, nbatch: nbatch, eval_metric: eval_metric, locals: Thread.CurrentThread.CurrentCulture); foreach (var call in batch_end_callback) { call(batch_end_params); } } if (epoch_size != null && nbatch >= epoch_size) { do_reset = false; break; } } if (do_reset) { logger.Info($"Epoch[{epoch}] Resetting Data Iterator"); train_data.Reset(); } if (epoch_size == null || nbatch >= epoch_size) { break; } } logger.Info($"Epoch[{epoch}] Time cost={(toc.ElapsedMilliseconds/1000):.000}"); } }