private void RunCore(IChannel ch)
{
Host.AssertValue(ch, "ch");
IDataSaver saver;
if (Args.Saver == null)
{
var ext = Path.GetExtension(Args.OutputDataFile);
var isBinary = string.Equals(ext, ".idv", StringComparison.OrdinalIgnoreCase);
var isTranspose = string.Equals(ext, ".tdv", StringComparison.OrdinalIgnoreCase);
if (isBinary)
{
saver = new BinarySaver(Host, new BinarySaver.Arguments());
}
else if (isTranspose)
{
saver = new TransposeSaver(Host, new TransposeSaver.Arguments());
}
else
{
saver = new TextSaver(Host, new TextSaver.Arguments());
}
}
else
{
saver = Args.Saver.CreateComponent(Host);
}
IDataLoader loader = CreateAndSaveLoader();
using (var file = Host.CreateOutputFile(Args.OutputDataFile))
DataSaverUtils.SaveDataView(ch, saver, loader, file, Args.KeepHidden);
}
public void TransposerSaverLoaderTest()
{
const int rowCount = 1000;
Random rgen = new Random(1);
ArrayDataViewBuilder builder = new ArrayDataViewBuilder(Env);
// A is to check the splitting of a sparse-ish column.
var dataA = GenerateHelper(rowCount, 0.1, rgen, () => (int)rgen.Next(), 50, 5, 10, 15);
dataA[rowCount / 2] = new VBuffer <int>(50, 0, null, null); // Coverage for the null vbuffer case.
builder.AddColumn("A", NumberDataViewType.Int32, dataA);
// B is to check the splitting of a dense-ish column.
builder.AddColumn("B", NumberDataViewType.Double, GenerateHelper(rowCount, 0.8, rgen, rgen.NextDouble, 50, 0, 25, 49));
// C is to just have some column we do nothing with.
builder.AddColumn("C", NumberDataViewType.Int16, GenerateHelper(rowCount, 0.1, rgen, () => (short)1, 30, 3, 10, 24));
// D is to check some column we don't have to split because it's sufficiently small.
builder.AddColumn("D", NumberDataViewType.Double, GenerateHelper(rowCount, 0.1, rgen, rgen.NextDouble, 3, 1));
// E is to check a sparse scalar column.
builder.AddColumn("E", NumberDataViewType.UInt32, GenerateHelper(rowCount, 0.1, rgen, () => (uint)rgen.Next(int.MinValue, int.MaxValue)));
// F is to check a dense-ish scalar column.
builder.AddColumn("F", NumberDataViewType.Int32, GenerateHelper(rowCount, 0.8, rgen, () => (int)rgen.Next()));
IDataView view = builder.GetDataView();
IMultiStreamSource src;
using (MemoryStream mem = new MemoryStream())
{
TransposeSaver saver = new TransposeSaver(Env, new TransposeSaver.Arguments());
saver.SaveData(mem, view, Utils.GetIdentityPermutation(view.Schema.Count));
src = new BytesStreamSource(mem.ToArray());
}
TransposeLoader loader = new TransposeLoader(Env, new TransposeLoader.Arguments(), src);
// First check whether this as an IDataView yields the same values.
CheckSameValues(view, loader);
TransposeCheckHelper <int>(view, 0, loader); // A
TransposeCheckHelper <Double>(view, 1, loader); // B
TransposeCheckHelper <short>(view, 2, loader); // C
TransposeCheckHelper <Double>(view, 3, loader); // D
TransposeCheckHelper <uint>(view, 4, loader); // E
TransposeCheckHelper <int>(view, 5, loader); // F
Done();
}
