public override bool Write(CTFBuilder builder)
{
if (_cursor == null)
{
_cursor = _data.GetEnumerator();
}
bool first = true;
for (var i = 0; i < _dimension; ++i)
{
if (_cursor.MoveNext() == false)
{
return(false);
}
if (first)
{
builder.AddDenseSample(_name);
}
first = false;
builder.AddDenseValue(_cursor.Current);
}
return(true);
}
public void Write(TextWriter writer)
{
var builder = new CTFBuilder(writer, 0, true);
var completed = new List <FeatureBase>();
while (true)
{
foreach (var f in _features)
{
if (completed.Contains(f))
{
continue;
}
if (f.Write(builder) == false)
{
completed.Add(f);
if (completed.Count == _features.Count)
{
builder.Finish();
return;
}
}
}
builder.NextLine();
}
}
protected override void EndProcessing()
{
Path = IO.GetAbsolutePath(this, Path);
var writer = new StreamWriter(Path, false, new UTF8Encoding(false));
var builder = new CTFBuilder(writer, InitialSequence, !ManualIncrement);
WriteObject(builder);
}
public override bool Write(CTFBuilder builder)
{
if (_cursor == null)
{
_cursor = _data.GetEnumerator();
}
if (_cursor.MoveNext() == false)
{
return(false);
}
builder.AddDenseSample(_name, _cursor.Current);
return(true);
}
public void Write(TextWriter writer)
{
var builder = new CTFBuilder(writer, 0, false);
int total = int.MaxValue;
foreach (var d in _featureMap)
{
total = Math.Min(total, d.Value.Count);
}
foreach (var d in _labelMap)
{
total = Math.Min(total, d.Value.Count);
}
for (var seq = 0; seq < total - _lookback + 1; ++seq)
{
for (var i = 0; i < _lookback; ++i)
{
foreach (var d in _featureMap)
{
builder.AddDenseSample(d.Key, new float[] { d.Value[seq + i] });
}
if (i == _lookback - 1)
{
foreach (var d in _labelMap)
{
builder.AddDenseSample(d.Key, new float[] { d.Value[seq + i] });
}
}
builder.NextLine();
}
builder.NextSequence();
}
builder.Finish();
}
public override bool Write(CTFBuilder builder)
{
if (_cursor == null)
{
_cursor = _data.GetEnumerator();
}
if (_cursor.MoveNext() == false)
{
return(false);
}
builder.AddSparseSample(_name);
var sample = _cursor.Current;
foreach (var e in sample)
{
builder.AddSparseValue(e.Item1, e.Item2);
}
return(true);
}
public abstract bool Write(CTFBuilder builder);
public static void Write(TextWriter writer, DataSourceSet dataSourceSet, bool withSequenceAxis)
{
var builder = new CTFBuilder(writer, 0, false);
// Argument check
var sampleCount = dataSourceSet.Features.First().Value.Shape[-1];
var maxSeqLength = 1;
foreach (var entry in dataSourceSet)
{
var name = entry.Key;
var ds = entry.Value;
if (withSequenceAxis && ds.Shape.Rank < 3)
{
throw new ArgumentException("DataSource shape should have sequence and batch axes as the last two");
}
if (!withSequenceAxis && ds.Shape.Rank < 2)
{
throw new ArgumentException("DataSource shape should have a batch axis");
}
var count = ds.Shape[-1];
if (count != sampleCount)
{
throw new ArgumentException("Sample counts of data sources should be equal");
}
if (withSequenceAxis)
{
var seqLength = ds.Shape[-2];
if (seqLength > maxSeqLength)
{
maxSeqLength = seqLength;
}
}
}
for (var sampleIndex = 0; sampleIndex < sampleCount; ++sampleIndex)
{
for (var seq = 0; seq < maxSeqLength; ++seq)
{
foreach (var entry in dataSourceSet)
{
var name = entry.Key;
var ds = entry.Value;
int seqLength;
int dim;
if (withSequenceAxis)
{
seqLength = ds.Shape[-2];
if (seq >= seqLength)
{
continue;
}
dim = ds.Shape.GetSize(ds.Shape.Rank - 3);
}
else
{
seqLength = 1;
dim = ds.Shape.GetSize(ds.Shape.Rank - 2);
}
int index = sampleIndex * dim * seqLength + seq * dim;
builder.AddDenseSample(name, new ListSlice <float>(ds.Data, index, dim));
}
builder.NextLine();
}
builder.NextSequence();
}
builder.Finish();
}
