public static CommonOutputs.TransformOutput SelectColumns(IHostEnvironment env, ScoreColumnSelectorInput input)
{
Contracts.CheckValue(env, nameof(env));
env.CheckValue(input, nameof(input));
EntryPointUtils.CheckInputArgs(env, input);
var view = input.Data;
var maxScoreId = view.Schema.GetMaxAnnotationKind(out int colMax, AnnotationUtils.Kinds.ScoreColumnSetId);
List <int> indices = new List <int>();
for (int i = 0; i < view.Schema.Count; i++)
{
if (view.Schema[i].IsHidden)
{
continue;
}
if (!ShouldAddColumn(view.Schema, i, input.ExtraColumns, maxScoreId))
{
continue;
}
indices.Add(i);
}
var newView = new ChooseColumnsByIndexTransform(env, new ChooseColumnsByIndexTransform.Options()
{
Indices = indices.ToArray()
}, input.Data);
return(new CommonOutputs.TransformOutput {
Model = new TransformModelImpl(env, newView, input.Data), OutputData = newView
});
}
private void SaveTransposedData(IChannel ch, Stream stream, ITransposeDataView data, int[] cols)
{
_host.AssertValue(ch);
ch.AssertValue(stream);
ch.AssertValue(data);
ch.AssertNonEmpty(cols);
ch.Assert(stream.CanSeek);
// Initialize what we can in the header, though we will not be writing out things in the
// header until we have confidence that things were written out correctly.
TransposeLoader.Header header = default(TransposeLoader.Header);
header.Signature = TransposeLoader.Header.SignatureValue;
header.Version = TransposeLoader.Header.WriterVersion;
header.CompatibleVersion = TransposeLoader.Header.WriterVersion;
VectorType slotType = data.TransposeSchema.GetSlotType(cols[0]);
ch.AssertValue(slotType);
header.RowCount = slotType.ValueCount;
header.ColumnCount = cols.Length;
// We keep track of the offsets of the start of each sub-IDV, for use in writing out the
// offsets/length table later.
List <long> offsets = new List <long>();
// First write a bunch of zeros at the head, as a placeholder for the header that
// will go there assuming we can successfully load it. We'll keep this array around
// for the real marshalling and writing of the header bytes structure.
byte[] headerBytes = new byte[TransposeLoader.Header.HeaderSize];
stream.Write(headerBytes, 0, headerBytes.Length);
offsets.Add(stream.Position);
// This is a convenient delegate to write out an IDV substream, then save the offsets
// where writing stopped to the offsets list.
Action <string, IDataView> viewAction =
(name, view) =>
{
using (var substream = new SubsetStream(stream))
{
_internalSaver.SaveData(substream, view, Utils.GetIdentityPermutation(view.Schema.ColumnCount));
substream.Seek(0, SeekOrigin.End);
ch.Info("Wrote {0} data view in {1} bytes", name, substream.Length);
}
offsets.Add(stream.Position);
};
// First write out the no-row data, limited to these columns.
IDataView subdata = new ChooseColumnsByIndexTransform(_host,
new ChooseColumnsByIndexTransform.Arguments()
{
Index = cols
}, data);
// If we want the "dual mode" row-wise and slot-wise file, don't filter out anything.
if (!_writeRowData)
{
subdata = SkipTakeFilter.Create(_host, new SkipTakeFilter.TakeArguments()
{
Count = 0
}, subdata);
}
string msg = _writeRowData ? "row-wise data, schema, and metadata" : "schema and metadata";
viewAction(msg, subdata);
foreach (var col in cols)
{
viewAction(data.Schema.GetColumnName(col), new TransposerUtils.SlotDataView(_host, data, col));
}
// Wrote out the dataview. Write out the table offset.
using (var writer = new BinaryWriter(stream, Encoding.UTF8, leaveOpen: true))
{
// Format of the table is offset, length, both as 8-byte integers.
// As it happens we wrote things out as adjacent sub-IDVs, so the
// length can be derived from the offsets. The first will be the
// start of the first sub-IDV, and all subsequent entries will be
// the start/end of the current/next sub-IDV, respectively, so a total
// of cols.Length + 2 entries.
ch.Assert(offsets.Count == cols.Length + 2);
ch.Assert(offsets[offsets.Count - 1] == stream.Position);
header.SubIdvTableOffset = stream.Position;
for (int c = 1; c < offsets.Count; ++c)
{
// 8-byte int for offsets, 8-byte int for length.
writer.Write(offsets[c - 1]);
writer.Write(offsets[c] - offsets[c - 1]);
}
header.TailOffset = stream.Position;
writer.Write(TransposeLoader.Header.TailSignatureValue);
// Now we are confident that things will work, so write it out.
unsafe
{
Marshal.Copy(new IntPtr(&header), headerBytes, 0, Marshal.SizeOf(typeof(Header)));
}
writer.Seek(0, SeekOrigin.Begin);
writer.Write(headerBytes);
}
}
