public virtual void Prepare(DataBatch data_batch, Func <DataBatch, NDArrayDict> sparse_row_id_fn = null) { if (sparse_row_id_fn != null) { Logger.Warning("sparse_row_id_fn is not invoked for BaseModule."); } }
private void CreateAndSendCorrespondence(string filename) { DataBatch batch = LoadBatch(filename); foreach (DataUnit dataunit in batch.DataUnits) { SendCorrespondenceDal.CreateAndSendCorrespondence(dataunit.archiveReference, dataunit.reportee); } }
public void SaveToCache(string _command, object _dataList) { DataBatch outValue; if (!Cache.TryGetValue(_command, out outValue)) { outValue = new DataBatch(); Cache.Add(_command, outValue); } outValue.DataSet = _dataList; }
public override void SetUp() { base.SetUp(); firstDataBatch = Presto.Persist <DataBatch>(); firstTenant = firstDataBatch.Tenant; secondDataBatch = Presto.Persist <DataBatch>(); secondTenant = secondDataBatch.Tenant; ClarityDB.Instance.SaveChanges(); ClarityDB.CreateInstance(firstDataBatch.UploadedBy); }
private void Start(int batchSize, int batchesCount, ITargetBlock <DataBatch <Person> > inputBlock) { for (var i = 0; i < batchesCount; i++) { var batch = new DataBatch <Person> { Number = i + 1, DataOffset = i * batchSize, IntendedSize = batchSize }; inputBlock.Post(batch); } inputBlock.Complete(); }
public void LoadNextBatchSet() { if (_currentEpoch >= 0) { for (int setIdx = 0; setIdx < Constants.BATCHES_PER_SET; setIdx++) { DataBatch batch; if ((Batches == null) || (Batches.Count < (Constants.BATCHES_PER_SET))) { batch = new DataBatch(Constants.TOTAL_VALUE_COUNT, 1, sparse: true); Batches.Add(batch); } else { batch = Batches[setIdx]; } for (var minibatchIdx = 0; minibatchIdx < Constants.MINIBATCH_SIZE; minibatchIdx++) { OneHotRecordReadOnly rec = _records[_recNo]; rec.CopyDataToSparseArray(batch.SparseIndices, batch.SparseValues, minibatchIdx); batch.Labels[minibatchIdx] = rec.Label; _recNo++; if (_recNo >= _records.Count) { _recNo = 0; if (_totalBatchCount == 0) { _totalBatchCount = _batchesRead; } _batchesRead = 0; _currentEpoch++; CurrentEpochBatch = 0; if (_shuffle) { _records.Shuffle(); } } _batchesRead++; } } } _currentBatch = 0; CurrentSet++; _loaded = true; }
public override void Forward(DataBatch data_batch, bool is_train = true) { if (!Binded && !ParamsInitialized) { throw new Exception("Module not binded and param initialized"); } var curr_data_shapes = _data_shapes.Select(x => x.Shape).ToArray(); var new_data_shapes = data_batch.Data.Select(x => x.Shape).ToArray(); if (curr_data_shapes.Length != new_data_shapes.Length) { DataDesc[] new_dshape; DataDesc[] new_lshape; if (data_batch.ProvideData != null) { new_dshape = data_batch.ProvideData; } else { new_dshape = _data_shapes.Zip(new_data_shapes, (i, shape) => { return(new DataDesc(i.Name, shape, i.DataType, i.Layout)); }).ToArray(); } if (data_batch.ProvideLabel != null) { new_lshape = data_batch.ProvideData; } else if (data_batch.Label != null) { new_lshape = _label_shapes.Zip(data_batch.Label, (i, j) => { return(new DataDesc(i.Name, j.Shape, i.DataType, i.Layout)); }).ToArray(); } else { new_lshape = null; } Reshape(new_dshape, new_lshape); } _exec_group.Forward(data_batch, is_train); }
public void LoadDataBatch(DataBatch data_batch) { if (sym_gen != null) { var key = data_batch.BucketKey.Value; if (execgrp_bucket.ContainsKey(key)) { symbol = sym_gen(key); execgrp = new DataParallelExecutorGroup(symbol, arg_names, param_names, contexts, slices, NDArrayIter.FromBatch(data_batch), execgrp); execgrp_bucket[key] = execgrp; } curr_execgrp = execgrp_bucket[key]; } else { curr_execgrp = execgrp; } curr_execgrp.LoadDataBatch(data_batch); }
public override void Prepare(DataBatch data_batch, Func <DataBatch, NDArrayDict> sparse_row_id_fn = null) { if (!Binded) { throw new Exception("Module not yet binded"); } if (sparse_row_id_fn != null) { if (_kvstore != null && !_update_on_kvstore.Value) { Logger.Warning("Parameters are not updated in the KVStore. " + "No need to call sparse_row_id_fn."); } else { var row_ids = sparse_row_id_fn(data_batch); foreach (var p in row_ids) { var param_name = p.Key; var row_id = p.Value; var param_idx = _exec_group.ParamNames.ToList().IndexOf(param_name); var param_val = _exec_group.ParamArrays[param_idx]; if (param_val[0].SType != StorageStype.RowSparse) { Logger.Warning($"{param_name}.stype is not 'row_sparse'. No need to " + "perform row_sparse_pull."); } else { _kvstore.RowSparsePull(param_name, param_val, row_ids: row_id, priority: param_idx); } } } } }
public void LoadDataBatch(DataBatch data_batch) { ExecuterManager.LoadData(data_batch, data_arrays.ToArray()); ExecuterManager.LoadData(data_batch, label_arrays.ToArray()); }
private int Batchify(DataBatch batch_data, DataBatch batch_label, int start = 0) { throw new NotImplementedException(); }
public void ForwardBackward(DataBatch data_batch) { Forward(data_batch); Backward(); }
internal static void LoadLabel(DataBatch batch, NDArrayList targets) { LoadGeneral(batch.Label, targets); }
internal static void LoadData(DataBatch batch, NDArrayList targets) { LoadGeneral(batch.Data, targets); }
internal static void LoadData(DataBatch batch, List <List <(Slice, NDArray)> > targets, int[] major_axis)
public void AddBatch(DataBatch batch) { dataBatches.Add(batch); }
public abstract void Forward(DataBatch data_batch, bool is_train = true);
public void ComputeLogLoss(int epochNo, int batchNo, bool test = true) { var provider = (test) ? TestProvider : TrainProvider; provider.LoadNextBatchSet(); var setNo = provider.CurrentSet; var correctPercentage = 0f; var recordCount = 0; var sum = 0d; DataBatch firstTestBatch = null; while (setNo == provider.CurrentSet) { _network.Calculate(train: !test); var testBatch = provider.CurrentBatch; if (firstTestBatch == null) { firstTestBatch = testBatch; } testBatch.PredictedLabels = _network.CostLayer.GetPredictedLabels(); if (testBatch == provider.Batches.Last()) { correctPercentage = provider.GetCorrectLabelPercentage(); } _network.CostLayer.Outputs.CopyToHost(); var eps = 1e-15; for (int i = 0; i < Constants.MINIBATCH_SIZE; i++) { var real = (double)testBatch.Labels[i]; var pred = (double)_network.CostLayer.Outputs[i, 1]; if (pred < eps) { pred = eps; } if (pred > (1d - eps)) { pred = 1d - eps; } var l = real * Math.Log(pred) + (1d - real) * Math.Log(1d - pred); sum += -l; recordCount++; } } var logLoss = sum / (double)recordCount; var averageLoss = _network.RegisterLoss(0, (float)logLoss, train: !test); var fNo = (firstTestBatch.EpocNo + "." + firstTestBatch.BatchNo).PadRight(10); var txt = (test) ? "Holdout" : "Train"; if (averageLoss < 0.44f) { Console.ForegroundColor = ConsoleColor.Green; } if (averageLoss > 0.44f) { Console.ForegroundColor = ConsoleColor.Yellow; } if (averageLoss > 0.45f) { Console.ForegroundColor = ConsoleColor.White; } if (averageLoss > 0.47f) { Console.ForegroundColor = ConsoleColor.Red; } if (averageLoss > 0.49f) { Console.ForegroundColor = ConsoleColor.DarkRed; } Console.WriteLine(txt + " " + fNo + "batch correct% : " + correctPercentage + " logloss last >1M recs : " + averageLoss); Console.ForegroundColor = ConsoleColor.White; }