private IDataSource <float>[] GetValues(Function func)
{
if (!_showValues)
{
return(null);
}
Value[] values = null;
try
{
if (_minibatch == null)
{
values = FunctionInvoke.Invoke(func, _arguments, null, false);
}
else
{
values = FunctionInvoke.Invoke(func, _minibatch, _map, null, false);
}
}
catch (Exception)
{
// Pass
}
return(values.Select(x => DataSourceFactory.FromValue(x)).ToArray());
}
public static string AsString(PSObject value, bool longFormat = true)
{
var v = value.BaseObject as CNTK.Value;
var ds = DataSourceFactory.FromValue(v);
return(Converter.ArrayToString <float>("CNTK.Value", ds.Data, ds.Shape, longFormat));
}
private void ProcessInternal <T>(Func <object, T> converter)
{
var ds = DataSourceFactory.FromPSObjects(_data, converter);
if (Dimensions != null)
{
ds.Reshape(Dimensions);
}
WriteObject(ds);
}
private static DataSourceSet ConvertFromSerializableObject(Dictionary <string, Tuple <float[], int[]> > obj)
{
var dss = new DataSourceSet();
foreach (var entry in obj)
{
dss.Add(entry.Key, DataSourceFactory.Create(entry.Value.Item1, entry.Value.Item2));
}
return(dss);
}
public NoiseSampler(string name, int[] shape, int minibatchSize, int iterationsPerEpoch, double min, double max, int?seed = null)
{
Name = name;
Shape = shape;
MinibatchSize = minibatchSize;
IterationsPerEpoch = iterationsPerEpoch;
Min = min;
Max = max;
_random = Random.GetInstance(seed);
_dataSize = Shape.GetSize(-1);
_data = new float[_dataSize * minibatchSize];
var minibatchShape = new int[Shape.Rank + 2];
shape.CopyTo(minibatchShape, 0);
minibatchShape[minibatchShape.Length - 2] = 1; // sequence
minibatchShape[minibatchShape.Length - 1] = minibatchSize; // sample
_samples = DataSourceFactory.Create(_data, minibatchShape);
Iterations = 0;
}
public bool ProcessVariable(Function holder, Variable va, int depth, bool visited)
{
if (visited)
{
return(true);
}
IDataSource <float>[] values = null;
if (va.Kind == VariableKind.Parameter || va.Kind == VariableKind.Constant)
{
values = new IDataSource <float>[] { DataSourceFactory.FromVariable(va) }
}
;
else if (va.Kind == VariableKind.Output)
{
values = GetValues(va.Owner);
}
var nodeInfo = new NodeInfo()
{
Node = (WrappedVariable)va,
Uid = va.Uid,
Name = va.Name,
Type = Utils.VariableKindName(va.Kind),
Path = GetPath(va.Uid, holder.Uid),
Parent = holder.Uid,
Children = va.Owner != null ? new string[] { va.Owner.Uid } : null,
Shape = va.Shape,
Values = values
};
_history.Add(va.Uid, nodeInfo);
_queue.Add(nodeInfo);
return(true);
}
public static IDataSource <float> VariableToDataSource(CNTK.Variable variable)
{
return(DataSourceFactory.FromVariable(variable));
}
public static IDataSource <float> ValueToDataSource(CNTK.Value value)
{
return(DataSourceFactory.FromValue(value));
}
public static IDataSource <float> ToDataSource(PSObject value)
{
var v = value.BaseObject as CNTK.Value;
return(DataSourceFactory.FromValue(v));
}
private void ProcessInternal <T>(Func <object, T> converter)
{
if (ParameterSetName == "load")
{
Path = IO.GetAbsolutePath(this, Path);
var result = DataSourceFactory.Load <float>(Path, !NoDecompress);
WriteObject(result);
}
else if (ParameterSetName == "rows")
{
var data = new List <T[]>();
foreach (var row in Rows)
{
var r = row.Select(x => {
if (x is PSObject)
{
x = (x as PSObject).BaseObject;
}
return(converter.Invoke(x));
});
data.Add(r.ToArray());
}
var result = DataSourceFactory.FromRows <T>(data, Dimensions);
WriteObject(result);
}
else if (ParameterSetName == "columns")
{
var data = new List <T[]>();
foreach (var column in Columns)
{
var c = column.Select(x => {
if (x is PSObject)
{
x = (x as PSObject).BaseObject;
}
return(converter.Invoke(x));
});
data.Add(c.ToArray());
}
var result = DataSourceFactory.FromColumns(data, Dimensions);
WriteObject(result);
}
else if (ParameterSetName == "psobjects")
{
var result = DataSourceFactory.FromPSObjects(PSObjects, converter);
if (Dimensions != null)
{
result.Reshape(Dimensions);
}
WriteObject(result);
}
else if (ParameterSetName == "datatable")
{
var result = DataSourceFactory.FromDataTable(DataTable, converter);
if (Dimensions != null)
{
result.Reshape(Dimensions);
}
WriteObject(result);
}
else
{
// new
var result = DataSourceFactory.Create(Data.Select(x => converter.Invoke(x)).ToArray(), Dimensions);
WriteObject(result);
}
}
public static IEnumerable <CTFSample> GetSampleReader(TextReader reader)
{
int lineCount = 0;
int sequenceCount = 0;
int seqStartLineCount = 1;
string line;
string seqId = null;
var splitLines = new List <string[]>();
var comments = new List <string>();
var startIndexMap = new Dictionary <string, int>();
var endIndexMap = new Dictionary <string, int>();
while ((line = reader.ReadLine()) != null)
{
++lineCount;
var splitLine = line.Split(new char[] { '|' });
var n = splitLine[0].Trim();
if (seqId == null || n == seqId)
{
splitLines.Add(splitLine);
seqId = n;
if (reader.Peek() != -1)
{
continue;
}
}
++sequenceCount;
var seqDim = splitLines.Count;
var dss = new DataSourceSet();
comments.Clear();
startIndexMap.Clear();
endIndexMap.Clear();
for (var i = 0; i < splitLines.Count; ++i)
{
var columns = splitLines[i];
for (var j = 1; j < columns.Length; ++j)
{
var feature = columns[j].Trim();
if (feature[0] == '#')
{
int skip;
for (skip = 1; skip < feature.Length && Char.IsWhiteSpace(feature[skip]); ++skip)
{
;
}
comments.Add(feature.Substring(skip));
continue;
}
var items = feature.Split();
if (items.Length < 2)
{
throw new InvalidDataException(string.Format("line {0}: Invalid feature", lineCount));
}
var featureDim = items.Length - 1;
var name = items[0];
DataSourceBase <float, float[]> ds;
float[] data;
if (dss.Features.ContainsKey(name))
{
ds = (DataSourceBase <float, float[]>)dss.Features[name];
data = ds.TypedData;
}
else
{
data = new float[featureDim * seqDim];
ds = DataSourceFactory.Create(data, new int[] { featureDim, seqDim, 1 });
dss.Add(name, ds);
startIndexMap[name] = i;
}
var baseIndex = ds.Shape.GetSequentialIndex(new int[] { 0, i, 0 });
for (var k = 0; k < featureDim; ++k)
{
data[baseIndex + k] = Converter.ToFloat(items[k + 1]);
}
endIndexMap[name] = i;
}
}
foreach (var name in dss.Features.Keys.ToArray())
{
var start = startIndexMap[name];
var end = endIndexMap[name];
if (start == 0 && end == splitLines.Count - 1)
{
continue;
}
var ds = dss[name];
dss.Features[name] = ds.Subset(start, end - start + 1, -2);
}
yield return(new CTFSample()
{
LineCount = seqStartLineCount,
SequenceCount = sequenceCount,
SequenceId = seqId,
DataSet = dss,
Comments = comments
});
splitLines.Clear();
splitLines.Add(splitLine);
seqStartLineCount = lineCount;
seqId = n;
}
}
