Ejemplo n.º 1
0
        private static void TrainMultiDevice(Symbol symbol,
                                             IList <Context> ctx,
                                             IList <string> argNames,
                                             IList <string> paramNames,
                                             IList <string> auxNames,
                                             Dictionary <string, NdArray> argParams,
                                             Dictionary <string, NdArray> auxParams,
                                             int beginEpoch,
                                             int endEpoch,
                                             int?epochSize,
                                             Optimizer optimizer,
                                             IDataIter trainData,
                                             IDataIter evalData,
                                             EvalMetric evalMetric,
                                             IList <EpochEndDelegate> epochEndCallback,
                                             IList <BatchEndDelegate> batchEndCallback,
                                             KvStore kvstore, bool updateOnKvstore,
                                             ILog logger,
                                             IList <int> workLoadList,
                                             Monitor monitor,
                                             IList <BatchEndDelegate> evalBatchEndCallback,
                                             SymbolGenerate symGen)
        {
            if (logger == null)
            {
                logger = LogManager.GetLogger("");
            }
            var executorManager = new DataParallelExecutorManager(symbol: symbol,
                                                                  symGen: symGen,
                                                                  ctx: ctx,
                                                                  trainData: trainData,
                                                                  paramNames: paramNames,
                                                                  argNames: argNames,
                                                                  auxNames: auxNames,
                                                                  workLoadList: workLoadList,
                                                                  logger: logger);


            if (monitor != null)
            {
                executorManager.InstallMonitor(monitor);
            }
            executorManager.SetParams(argParams, auxParams);

            Action <int, NdArray, NdArray> updater = null;

            if (!updateOnKvstore)
            {
                updater = Optimizer.GetUpdater(optimizer);
            }
            if (kvstore != null)
            {
                InitializeKvstore(kvstore: kvstore,
                                  paramArrays: executorManager.ParamArrays,
                                  argParams: argParams,
                                  paramNames: executorManager.ParamNames,
                                  updateOnKvstore: updateOnKvstore);
            }

            if (updateOnKvstore)
            {
                kvstore?.SetOptimizer(optimizer);
            }

            //Now start training
            for (int epoch = 0; epoch < endEpoch - beginEpoch; epoch++)
            {
                // Training phase
                Stopwatch toc = new Stopwatch();
                toc.Start();
                evalMetric.Reset();
                var nbatch = 0;
                // Iterate over training data.

                while (true)
                {
                    var doReset = true;
                    foreach (var dataBatch in trainData)
                    {
                        executorManager.LoadDataBatch(dataBatch);

                        monitor?.Tic();


                        executorManager.Forward(isTrain: true);
                        executorManager.Backward();



                        if (updateOnKvstore)
                        {
                            UpdateParamsOnKvstore(
                                executorManager.ParamArrays,
                                executorManager.GradArrays,
                                kvstore);
                        }
                        else
                        {
                            UpdateParams(executorManager.ParamArrays,
                                         executorManager.GradArrays,
                                         updater: updater,
                                         numDevice: ctx.Count,
                                         kvstore: kvstore);
                        }
                        monitor?.TocPrint();
                        // evaluate at end, so we can lazy copy
                        executorManager.UpdateMetric(evalMetric, dataBatch.Label);

                        nbatch += 1;
                        //batch callback (for print purpose)

                        if (batchEndCallback != null)
                        {
                            var batchEndParams = new BatchEndParam(epoch: epoch,
                                                                   nbatch: nbatch,
                                                                   evalMetric: evalMetric,
                                                                   locals: Thread.CurrentThread.CurrentCulture);

                            foreach (var call in batchEndCallback)
                            {
                                call(batchEndParams);
                            }
                        }
                        if (epochSize != null && nbatch >= epochSize)
                        {
                            doReset = false;
                            break;
                        }
                    }

                    if (doReset)
                    {
                        logger.Info($"Epoch[{epoch}] Resetting Data Iterator");
                        trainData.Reset();
                    }

                    if (epochSize == null || nbatch >= epochSize)
                    {
                        break;
                    }
                }


                logger.Info($"Epoch[{epoch}] Time cost={(toc.ElapsedMilliseconds/1000):.000}");

                if (epochEndCallback != null || epoch + 1 == endEpoch)
                {
                    executorManager.copy_to(argParams, auxParams);
                }


                if (epochEndCallback != null)
                {
                    EpochEndParam epochEndParam = new EpochEndParam(epoch, symbol, argParams, auxParams);

                    foreach (var callitem in epochEndCallback)
                    {
                        callitem(epochEndParam);
                    }
                }

                // evaluation
                if (evalData != null)
                {
                    evalMetric.Reset();
                    evalData.Reset();
                    int i = 0;
                    foreach (var eval_batch in evalData)
                    {
                        executorManager.LoadDataBatch(eval_batch);
                        executorManager.Forward(isTrain: false);
                        executorManager.UpdateMetric(evalMetric, eval_batch.Label);

                        if (evalBatchEndCallback != null)
                        {
                            var batchEndParams = new BatchEndParam(epoch: epoch,
                                                                   nbatch: i,
                                                                   evalMetric: evalMetric,
                                                                   locals: Thread.CurrentThread.CurrentCulture);
                            foreach (var call in evalBatchEndCallback)
                            {
                                call(batchEndParams);
                            }
                        }

                        i++;
                    }
                    var nameValue = evalMetric.get_name_value();
                    foreach (var item in nameValue)
                    {
                        logger.Info($"Epoch[{epoch}] Validation-{item.Name}={item.Value:0.000}");
                    }
                    evalData.Reset();
                }
            }
        }
Ejemplo n.º 2
0
        private static void _train_multi_device(Symbol symbol, List <Context> ctx, List <string> arg_names,
                                                List <string> param_names, List <string> aux_names, Dictionary <string, NDArray> arg_params,
                                                Dictionary <string, NDArray> aux_params, int begin_epoch, int end_epoch, int?epoch_size, Optimizer optimizer,
                                                IDataIter train_data, IDataIter eval_data, EvalMetric eval_metric, List <Action> epoch_end_callback,
                                                List <Action <BatchEndParam> > batch_end_callback, KVStore kvstore, bool update_on_kvstore, ILog logger, List <int> work_load_list,
                                                Monitor monitor, Action eval_batch_end_callback, SymbolGenerate sym_gen)
        {
            if (logger == null)
            {
                logger = LogManager.GetLogger("");
            }
            var executor_manager = new DataParallelExecutorManager(symbol: symbol,
                                                                   sym_gen: sym_gen,
                                                                   ctx: ctx,
                                                                   train_data: train_data,
                                                                   param_names: param_names,
                                                                   arg_names: arg_names,
                                                                   aux_names: aux_names,
                                                                   work_load_list: work_load_list,
                                                                   logger: logger);


            if (monitor != null)
            {
                executor_manager.install_monitor(monitor);
            }
            executor_manager.set_params(arg_params, aux_params);

            Action <int, NDArray, NDArray> updater = null;

            if (!update_on_kvstore)
            {
                updater = Optimizer.get_updater(optimizer);
            }
            if (kvstore != null)
            {
                _initialize_kvstore(kvstore: kvstore,
                                    param_arrays: executor_manager.param_arrays,
                                    arg_params: arg_params,
                                    param_names: executor_manager.param_names,
                                    update_on_kvstore: update_on_kvstore);
            }

            if (update_on_kvstore)
            {
                kvstore.set_optimizer(optimizer);
            }

            //Now start training
            for (int epoch = 0; epoch < end_epoch - begin_epoch; epoch++)
            {
                // Training phase
                Stopwatch toc = new Stopwatch();
                toc.Start();
                eval_metric.Reset();
                var nbatch = 0;
                // Iterate over training data.

                while (true)
                {
                    var do_reset = true;
                    foreach (var data_batch in train_data)
                    {
                        executor_manager.load_data_batch(data_batch);

                        monitor?.Tic();


                        executor_manager.Forward(is_train: true);
                        executor_manager.Backward();



                        if (update_on_kvstore)
                        {
                            _update_params_on_kvstore(
                                executor_manager.param_arrays,
                                executor_manager.grad_arrays,
                                kvstore);
                        }
                        else
                        {
                            _update_params(executor_manager.param_arrays,
                                           executor_manager.grad_arrays,
                                           updater: updater,
                                           num_device: ctx.Count,
                                           kvstore: kvstore);
                        }
                        monitor?.toc_print();
                        // evaluate at end, so we can lazy copy
                        executor_manager.update_metric(eval_metric, data_batch.label);

                        nbatch += 1;
                        //batch callback (for print purpose)

                        if (batch_end_callback != null)
                        {
                            var batch_end_params = new BatchEndParam(epoch: epoch,
                                                                     nbatch: nbatch,
                                                                     eval_metric: eval_metric,
                                                                     locals: Thread.CurrentThread.CurrentCulture);

                            foreach (var call in batch_end_callback)
                            {
                                call(batch_end_params);
                            }
                        }
                        if (epoch_size != null && nbatch >= epoch_size)
                        {
                            do_reset = false;
                            break;
                        }
                    }

                    if (do_reset)
                    {
                        logger.Info($"Epoch[{epoch}] Resetting Data Iterator");
                        train_data.Reset();
                    }

                    if (epoch_size == null || nbatch >= epoch_size)
                    {
                        break;
                    }
                }


                logger.Info($"Epoch[{epoch}] Time cost={(toc.ElapsedMilliseconds/1000):.000}");
            }
        }