public virtual void Prepare(DataBatch data_batch, Func <DataBatch, NDArrayDict> sparse_row_id_fn = null)
{
if (sparse_row_id_fn != null)
{
Logger.Warning("sparse_row_id_fn is not invoked for BaseModule.");
}
}
private void CreateAndSendCorrespondence(string filename)
{
DataBatch batch = LoadBatch(filename);
foreach (DataUnit dataunit in batch.DataUnits)
{
SendCorrespondenceDal.CreateAndSendCorrespondence(dataunit.archiveReference, dataunit.reportee);
}
}
public void SaveToCache(string _command, object _dataList)
{
DataBatch outValue;
if (!Cache.TryGetValue(_command, out outValue))
{
outValue = new DataBatch();
Cache.Add(_command, outValue);
}
outValue.DataSet = _dataList;
}
public override void SetUp()
{
base.SetUp();
firstDataBatch = Presto.Persist <DataBatch>();
firstTenant = firstDataBatch.Tenant;
secondDataBatch = Presto.Persist <DataBatch>();
secondTenant = secondDataBatch.Tenant;
ClarityDB.Instance.SaveChanges();
ClarityDB.CreateInstance(firstDataBatch.UploadedBy);
}
private void Start(int batchSize, int batchesCount, ITargetBlock <DataBatch <Person> > inputBlock)
{
for (var i = 0; i < batchesCount; i++)
{
var batch = new DataBatch <Person>
{
Number = i + 1,
DataOffset = i * batchSize,
IntendedSize = batchSize
};
inputBlock.Post(batch);
}
inputBlock.Complete();
}
public void LoadNextBatchSet()
{
if (_currentEpoch >= 0)
{
for (int setIdx = 0; setIdx < Constants.BATCHES_PER_SET; setIdx++)
{
DataBatch batch;
if ((Batches == null) || (Batches.Count < (Constants.BATCHES_PER_SET)))
{
batch = new DataBatch(Constants.TOTAL_VALUE_COUNT, 1, sparse: true);
Batches.Add(batch);
}
else
{
batch = Batches[setIdx];
}
for (var minibatchIdx = 0; minibatchIdx < Constants.MINIBATCH_SIZE; minibatchIdx++)
{
OneHotRecordReadOnly rec = _records[_recNo];
rec.CopyDataToSparseArray(batch.SparseIndices, batch.SparseValues, minibatchIdx);
batch.Labels[minibatchIdx] = rec.Label;
_recNo++;
if (_recNo >= _records.Count)
{
_recNo = 0;
if (_totalBatchCount == 0)
{
_totalBatchCount = _batchesRead;
}
_batchesRead = 0;
_currentEpoch++;
CurrentEpochBatch = 0;
if (_shuffle)
{
_records.Shuffle();
}
}
_batchesRead++;
}
}
}
_currentBatch = 0;
CurrentSet++;
_loaded = true;
}
public override void Forward(DataBatch data_batch, bool is_train = true)
{
if (!Binded && !ParamsInitialized)
{
throw new Exception("Module not binded and param initialized");
}
var curr_data_shapes = _data_shapes.Select(x => x.Shape).ToArray();
var new_data_shapes = data_batch.Data.Select(x => x.Shape).ToArray();
if (curr_data_shapes.Length != new_data_shapes.Length)
{
DataDesc[] new_dshape;
DataDesc[] new_lshape;
if (data_batch.ProvideData != null)
{
new_dshape = data_batch.ProvideData;
}
else
{
new_dshape = _data_shapes.Zip(new_data_shapes,
(i, shape) => { return(new DataDesc(i.Name, shape, i.DataType, i.Layout)); }).ToArray();
}
if (data_batch.ProvideLabel != null)
{
new_lshape = data_batch.ProvideData;
}
else if (data_batch.Label != null)
{
new_lshape = _label_shapes.Zip(data_batch.Label,
(i, j) => { return(new DataDesc(i.Name, j.Shape, i.DataType, i.Layout)); }).ToArray();
}
else
{
new_lshape = null;
}
Reshape(new_dshape, new_lshape);
}
_exec_group.Forward(data_batch, is_train);
}
public void LoadDataBatch(DataBatch data_batch)
{
if (sym_gen != null)
{
var key = data_batch.BucketKey.Value;
if (execgrp_bucket.ContainsKey(key))
{
symbol = sym_gen(key);
execgrp = new DataParallelExecutorGroup(symbol, arg_names, param_names, contexts, slices,
NDArrayIter.FromBatch(data_batch), execgrp);
execgrp_bucket[key] = execgrp;
}
curr_execgrp = execgrp_bucket[key];
}
else
{
curr_execgrp = execgrp;
}
curr_execgrp.LoadDataBatch(data_batch);
}
public override void Prepare(DataBatch data_batch, Func <DataBatch, NDArrayDict> sparse_row_id_fn = null)
{
if (!Binded)
{
throw new Exception("Module not yet binded");
}
if (sparse_row_id_fn != null)
{
if (_kvstore != null && !_update_on_kvstore.Value)
{
Logger.Warning("Parameters are not updated in the KVStore. " +
"No need to call sparse_row_id_fn.");
}
else
{
var row_ids = sparse_row_id_fn(data_batch);
foreach (var p in row_ids)
{
var param_name = p.Key;
var row_id = p.Value;
var param_idx = _exec_group.ParamNames.ToList().IndexOf(param_name);
var param_val = _exec_group.ParamArrays[param_idx];
if (param_val[0].SType != StorageStype.RowSparse)
{
Logger.Warning($"{param_name}.stype is not 'row_sparse'. No need to " +
"perform row_sparse_pull.");
}
else
{
_kvstore.RowSparsePull(param_name, param_val, row_ids: row_id, priority: param_idx);
}
}
}
}
}
public void LoadDataBatch(DataBatch data_batch)
{
ExecuterManager.LoadData(data_batch, data_arrays.ToArray());
ExecuterManager.LoadData(data_batch, label_arrays.ToArray());
}
private int Batchify(DataBatch batch_data, DataBatch batch_label, int start = 0)
{
throw new NotImplementedException();
}
public void ForwardBackward(DataBatch data_batch)
{
Forward(data_batch);
Backward();
}
internal static void LoadLabel(DataBatch batch, NDArrayList targets)
{
LoadGeneral(batch.Label, targets);
}
internal static void LoadData(DataBatch batch, NDArrayList targets)
{
LoadGeneral(batch.Data, targets);
}
internal static void LoadData(DataBatch batch, List <List <(Slice, NDArray)> > targets, int[] major_axis)
public void AddBatch(DataBatch batch)
{
dataBatches.Add(batch);
}
public abstract void Forward(DataBatch data_batch, bool is_train = true);
public void ComputeLogLoss(int epochNo, int batchNo, bool test = true)
{
var provider = (test) ? TestProvider : TrainProvider;
provider.LoadNextBatchSet();
var setNo = provider.CurrentSet;
var correctPercentage = 0f;
var recordCount = 0;
var sum = 0d;
DataBatch firstTestBatch = null;
while (setNo == provider.CurrentSet)
{
_network.Calculate(train: !test);
var testBatch = provider.CurrentBatch;
if (firstTestBatch == null)
{
firstTestBatch = testBatch;
}
testBatch.PredictedLabels = _network.CostLayer.GetPredictedLabels();
if (testBatch == provider.Batches.Last())
{
correctPercentage = provider.GetCorrectLabelPercentage();
}
_network.CostLayer.Outputs.CopyToHost();
var eps = 1e-15;
for (int i = 0; i < Constants.MINIBATCH_SIZE; i++)
{
var real = (double)testBatch.Labels[i];
var pred = (double)_network.CostLayer.Outputs[i, 1];
if (pred < eps)
{
pred = eps;
}
if (pred > (1d - eps))
{
pred = 1d - eps;
}
var l = real * Math.Log(pred) + (1d - real) * Math.Log(1d - pred);
sum += -l;
recordCount++;
}
}
var logLoss = sum / (double)recordCount;
var averageLoss = _network.RegisterLoss(0, (float)logLoss, train: !test);
var fNo = (firstTestBatch.EpocNo + "." + firstTestBatch.BatchNo).PadRight(10);
var txt = (test) ? "Holdout" : "Train";
if (averageLoss < 0.44f)
{
Console.ForegroundColor = ConsoleColor.Green;
}
if (averageLoss > 0.44f)
{
Console.ForegroundColor = ConsoleColor.Yellow;
}
if (averageLoss > 0.45f)
{
Console.ForegroundColor = ConsoleColor.White;
}
if (averageLoss > 0.47f)
{
Console.ForegroundColor = ConsoleColor.Red;
}
if (averageLoss > 0.49f)
{
Console.ForegroundColor = ConsoleColor.DarkRed;
}
Console.WriteLine(txt + " " + fNo + "batch correct% : " + correctPercentage + " logloss last >1M recs : " + averageLoss);
Console.ForegroundColor = ConsoleColor.White;
}
