private void TryDiscardSnapshots(IDataBatch batch)
{
foreach (var slice in batch.ParticipatingSlices)
{
if (slicesTracker.TryCompleteSnapshot(slice))
{
slice.Buffer.DiscardSnapshot();
}
}
}
private void LogBatchSendResult(IDataBatch batch, RequestSendResult result, TimeSpan elapsed)
{
if (result == RequestSendResult.Success)
{
log.Info($"Successfully sent batch of size {batch.SerializedMessage.Count.Bytes()} in {elapsed}.");
}
else
{
log.Warn($"Failed to send batch of size {batch.SerializedMessage.Count.Bytes()} with result '{result}' in {elapsed}.");
}
}
public void LoadDataBatch(IDataBatch dataBatch)
{
if (this._symGen != null)
{
var key = dataBatch.BucketKey;
if (!_execgrpBucket.ContainsKey(key))
{
//create new bucket entry
var symbol = _symGen(key);
var execgrp = new DataParallelExecutorGroup(symbol, this._argNames,
this.ParamNames, this._ctx,
this._slices, dataBatch,
sharedGroup: this._execgrp);
this._execgrpBucket[key] = execgrp;
}
this._currExecgrp = this._execgrpBucket[key];
}
else
{
this._currExecgrp = this._execgrp;
}
this._currExecgrp.LoadDataBatch(dataBatch);
}
public void load_data_batch(IDataBatch data_batch)
{
if (this._sym_gen != null)
{
var key = data_batch.bucket_key;
if (!_execgrp_bucket.ContainsKey(key))
{
//create new bucket entry
var symbol = _sym_gen(key);
var execgrp = new DataParallelExecutorGroup(symbol, this._arg_names,
this.param_names, this._ctx,
this._slices, data_batch,
shared_group: this._execgrp);
this._execgrp_bucket[key] = execgrp;
}
this._curr_execgrp = this._execgrp_bucket[key];
}
else
{
this._curr_execgrp = this._execgrp;
}
this._curr_execgrp.load_data_batch(data_batch);
}
public static void _load_label(IDataBatch batch, IList <List <Tuple <Tuple <int, int>, NdArray> > > targets)
{
Load_general(batch.Label, targets);
}
public static void _load_label(IDataBatch batch, IList <NdArray> targets)
{
Load_general(batch.Label, targets);
}
public static void LoadData(IDataBatch batch, IList <List <Tuple <Tuple <int, int>, NdArray> > > targets)
{
Load_general(batch.Data, targets);
}
public static void LoadData(IDataBatch batch, IList <NdArray> targets)
{
Load_general(batch.Data, targets);
}
public void LoadDataBatch(IDataBatch dataBatch)
{
ExecutorManager.LoadData(dataBatch, this._dataArrays);
ExecutorManager._load_label(dataBatch, this._labelArrays);
}
private static void _load_label(IDataBatch batch, List <List <Tuple <Tuple <int, int>, NDArray> > > targets)
{
_load_general(batch.label, targets);
}
public void load_data_batch(IDataBatch data_batch)
{
_load_data(data_batch, this._data_arrays);
_load_label(data_batch, this._label_arrays);
}
