private IDataSource <float>[] GetValues(Function func) { if (!_showValues) { return(null); } Value[] values = null; try { if (_minibatch == null) { values = FunctionInvoke.Invoke(func, _arguments, null, false); } else { values = FunctionInvoke.Invoke(func, _minibatch, _map, null, false); } } catch (Exception) { // Pass } return(values.Select(x => DataSourceFactory.FromValue(x)).ToArray()); }
public static string AsString(PSObject value, bool longFormat = true) { var v = value.BaseObject as CNTK.Value; var ds = DataSourceFactory.FromValue(v); return(Converter.ArrayToString <float>("CNTK.Value", ds.Data, ds.Shape, longFormat)); }
private void ProcessInternal <T>(Func <object, T> converter) { var ds = DataSourceFactory.FromPSObjects(_data, converter); if (Dimensions != null) { ds.Reshape(Dimensions); } WriteObject(ds); }
private static DataSourceSet ConvertFromSerializableObject(Dictionary <string, Tuple <float[], int[]> > obj) { var dss = new DataSourceSet(); foreach (var entry in obj) { dss.Add(entry.Key, DataSourceFactory.Create(entry.Value.Item1, entry.Value.Item2)); } return(dss); }
public NoiseSampler(string name, int[] shape, int minibatchSize, int iterationsPerEpoch, double min, double max, int?seed = null) { Name = name; Shape = shape; MinibatchSize = minibatchSize; IterationsPerEpoch = iterationsPerEpoch; Min = min; Max = max; _random = Random.GetInstance(seed); _dataSize = Shape.GetSize(-1); _data = new float[_dataSize * minibatchSize]; var minibatchShape = new int[Shape.Rank + 2]; shape.CopyTo(minibatchShape, 0); minibatchShape[minibatchShape.Length - 2] = 1; // sequence minibatchShape[minibatchShape.Length - 1] = minibatchSize; // sample _samples = DataSourceFactory.Create(_data, minibatchShape); Iterations = 0; }
public bool ProcessVariable(Function holder, Variable va, int depth, bool visited) { if (visited) { return(true); } IDataSource <float>[] values = null; if (va.Kind == VariableKind.Parameter || va.Kind == VariableKind.Constant) { values = new IDataSource <float>[] { DataSourceFactory.FromVariable(va) } } ; else if (va.Kind == VariableKind.Output) { values = GetValues(va.Owner); } var nodeInfo = new NodeInfo() { Node = (WrappedVariable)va, Uid = va.Uid, Name = va.Name, Type = Utils.VariableKindName(va.Kind), Path = GetPath(va.Uid, holder.Uid), Parent = holder.Uid, Children = va.Owner != null ? new string[] { va.Owner.Uid } : null, Shape = va.Shape, Values = values }; _history.Add(va.Uid, nodeInfo); _queue.Add(nodeInfo); return(true); }
public static IDataSource <float> VariableToDataSource(CNTK.Variable variable) { return(DataSourceFactory.FromVariable(variable)); }
public static IDataSource <float> ValueToDataSource(CNTK.Value value) { return(DataSourceFactory.FromValue(value)); }
public static IDataSource <float> ToDataSource(PSObject value) { var v = value.BaseObject as CNTK.Value; return(DataSourceFactory.FromValue(v)); }
private void ProcessInternal <T>(Func <object, T> converter) { if (ParameterSetName == "load") { Path = IO.GetAbsolutePath(this, Path); var result = DataSourceFactory.Load <float>(Path, !NoDecompress); WriteObject(result); } else if (ParameterSetName == "rows") { var data = new List <T[]>(); foreach (var row in Rows) { var r = row.Select(x => { if (x is PSObject) { x = (x as PSObject).BaseObject; } return(converter.Invoke(x)); }); data.Add(r.ToArray()); } var result = DataSourceFactory.FromRows <T>(data, Dimensions); WriteObject(result); } else if (ParameterSetName == "columns") { var data = new List <T[]>(); foreach (var column in Columns) { var c = column.Select(x => { if (x is PSObject) { x = (x as PSObject).BaseObject; } return(converter.Invoke(x)); }); data.Add(c.ToArray()); } var result = DataSourceFactory.FromColumns(data, Dimensions); WriteObject(result); } else if (ParameterSetName == "psobjects") { var result = DataSourceFactory.FromPSObjects(PSObjects, converter); if (Dimensions != null) { result.Reshape(Dimensions); } WriteObject(result); } else if (ParameterSetName == "datatable") { var result = DataSourceFactory.FromDataTable(DataTable, converter); if (Dimensions != null) { result.Reshape(Dimensions); } WriteObject(result); } else { // new var result = DataSourceFactory.Create(Data.Select(x => converter.Invoke(x)).ToArray(), Dimensions); WriteObject(result); } }
public static IEnumerable <CTFSample> GetSampleReader(TextReader reader) { int lineCount = 0; int sequenceCount = 0; int seqStartLineCount = 1; string line; string seqId = null; var splitLines = new List <string[]>(); var comments = new List <string>(); var startIndexMap = new Dictionary <string, int>(); var endIndexMap = new Dictionary <string, int>(); while ((line = reader.ReadLine()) != null) { ++lineCount; var splitLine = line.Split(new char[] { '|' }); var n = splitLine[0].Trim(); if (seqId == null || n == seqId) { splitLines.Add(splitLine); seqId = n; if (reader.Peek() != -1) { continue; } } ++sequenceCount; var seqDim = splitLines.Count; var dss = new DataSourceSet(); comments.Clear(); startIndexMap.Clear(); endIndexMap.Clear(); for (var i = 0; i < splitLines.Count; ++i) { var columns = splitLines[i]; for (var j = 1; j < columns.Length; ++j) { var feature = columns[j].Trim(); if (feature[0] == '#') { int skip; for (skip = 1; skip < feature.Length && Char.IsWhiteSpace(feature[skip]); ++skip) { ; } comments.Add(feature.Substring(skip)); continue; } var items = feature.Split(); if (items.Length < 2) { throw new InvalidDataException(string.Format("line {0}: Invalid feature", lineCount)); } var featureDim = items.Length - 1; var name = items[0]; DataSourceBase <float, float[]> ds; float[] data; if (dss.Features.ContainsKey(name)) { ds = (DataSourceBase <float, float[]>)dss.Features[name]; data = ds.TypedData; } else { data = new float[featureDim * seqDim]; ds = DataSourceFactory.Create(data, new int[] { featureDim, seqDim, 1 }); dss.Add(name, ds); startIndexMap[name] = i; } var baseIndex = ds.Shape.GetSequentialIndex(new int[] { 0, i, 0 }); for (var k = 0; k < featureDim; ++k) { data[baseIndex + k] = Converter.ToFloat(items[k + 1]); } endIndexMap[name] = i; } } foreach (var name in dss.Features.Keys.ToArray()) { var start = startIndexMap[name]; var end = endIndexMap[name]; if (start == 0 && end == splitLines.Count - 1) { continue; } var ds = dss[name]; dss.Features[name] = ds.Subset(start, end - start + 1, -2); } yield return(new CTFSample() { LineCount = seqStartLineCount, SequenceCount = sequenceCount, SequenceId = seqId, DataSet = dss, Comments = comments }); splitLines.Clear(); splitLines.Add(splitLine); seqStartLineCount = lineCount; seqId = n; } }