private void RunCore(IChannel ch) { Host.AssertValue(ch, "ch"); IDataSaver saver; if (Args.Saver == null) { var ext = Path.GetExtension(Args.OutputDataFile); var isBinary = string.Equals(ext, ".idv", StringComparison.OrdinalIgnoreCase); var isTranspose = string.Equals(ext, ".tdv", StringComparison.OrdinalIgnoreCase); if (isBinary) { saver = new BinarySaver(Host, new BinarySaver.Arguments()); } else if (isTranspose) { saver = new TransposeSaver(Host, new TransposeSaver.Arguments()); } else { saver = new TextSaver(Host, new TextSaver.Arguments()); } } else { saver = Args.Saver.CreateComponent(Host); } IDataLoader loader = CreateAndSaveLoader(); using (var file = Host.CreateOutputFile(Args.OutputDataFile)) DataSaverUtils.SaveDataView(ch, saver, loader, file, Args.KeepHidden); }
public void TransposerSaverLoaderTest() { const int rowCount = 1000; Random rgen = new Random(1); ArrayDataViewBuilder builder = new ArrayDataViewBuilder(Env); // A is to check the splitting of a sparse-ish column. var dataA = GenerateHelper(rowCount, 0.1, rgen, () => (int)rgen.Next(), 50, 5, 10, 15); dataA[rowCount / 2] = new VBuffer <int>(50, 0, null, null); // Coverage for the null vbuffer case. builder.AddColumn("A", NumberDataViewType.Int32, dataA); // B is to check the splitting of a dense-ish column. builder.AddColumn("B", NumberDataViewType.Double, GenerateHelper(rowCount, 0.8, rgen, rgen.NextDouble, 50, 0, 25, 49)); // C is to just have some column we do nothing with. builder.AddColumn("C", NumberDataViewType.Int16, GenerateHelper(rowCount, 0.1, rgen, () => (short)1, 30, 3, 10, 24)); // D is to check some column we don't have to split because it's sufficiently small. builder.AddColumn("D", NumberDataViewType.Double, GenerateHelper(rowCount, 0.1, rgen, rgen.NextDouble, 3, 1)); // E is to check a sparse scalar column. builder.AddColumn("E", NumberDataViewType.UInt32, GenerateHelper(rowCount, 0.1, rgen, () => (uint)rgen.Next(int.MinValue, int.MaxValue))); // F is to check a dense-ish scalar column. builder.AddColumn("F", NumberDataViewType.Int32, GenerateHelper(rowCount, 0.8, rgen, () => (int)rgen.Next())); IDataView view = builder.GetDataView(); IMultiStreamSource src; using (MemoryStream mem = new MemoryStream()) { TransposeSaver saver = new TransposeSaver(Env, new TransposeSaver.Arguments()); saver.SaveData(mem, view, Utils.GetIdentityPermutation(view.Schema.Count)); src = new BytesStreamSource(mem.ToArray()); } TransposeLoader loader = new TransposeLoader(Env, new TransposeLoader.Arguments(), src); // First check whether this as an IDataView yields the same values. CheckSameValues(view, loader); TransposeCheckHelper <int>(view, 0, loader); // A TransposeCheckHelper <Double>(view, 1, loader); // B TransposeCheckHelper <short>(view, 2, loader); // C TransposeCheckHelper <Double>(view, 3, loader); // D TransposeCheckHelper <uint>(view, 4, loader); // E TransposeCheckHelper <int>(view, 5, loader); // F Done(); }