public TransposeLoader(IHostEnvironment env, Arguments args, IMultiStreamSource file)
{
Contracts.CheckValue(env, nameof(env));
_host = env.Register(LoadName);
_host.CheckValue(args, nameof(args));
_host.CheckValue(file, nameof(file));
_host.Check(file.Count == 1, "Transposed loader accepts a single file only");
_threads = args.Threads ?? 0;
if (_threads < 0)
{
_threads = 0;
}
_file = file;
using (Stream stream = _file.Open(0))
using (BinaryReader reader = new BinaryReader(stream))
{
_header = InitHeader(reader);
reader.Seek(_header.SubIdvTableOffset);
_schemaEntry = new SubIdvEntry.SchemaSubIdv(this, reader);
_entries = new SubIdvEntry.TransposedSubIdv[_header.ColumnCount];
for (int c = 0; c < _entries.Length; ++c)
{
_entries[c] = new SubIdvEntry.TransposedSubIdv(this, reader, c);
}
_schema = new SchemaImpl(this);
if (!HasRowData)
{
_colTransposers = new Transposer[_header.ColumnCount];
_colTransposersLock = new object();
}
}
}
private TransposeLoader(IHost host, ModelLoadContext ctx, IMultiStreamSource file)
{
Contracts.CheckValue(host, nameof(host));
_host = host;
_host.CheckValue(file, nameof(file));
_host.Check(file.Count == 1, "Transposed loader accepts a single file only");
// *** Binary format **
// int: Number of threads if explicitly defined, or 0 if the
// number of threads was automatically determined
_threads = ctx.Reader.ReadInt32();
_host.CheckDecode(_threads >= 0);
// Dedupe code somehow?
_file = file;
using (Stream stream = _file.Open(0))
using (BinaryReader reader = new BinaryReader(stream))
{
_header = InitHeader(reader);
reader.Seek(_header.SubIdvTableOffset);
_schemaEntry = new SubIdvEntry.SchemaSubIdv(this, reader);
_entries = new SubIdvEntry.TransposedSubIdv[_header.ColumnCount];
for (int c = 0; c < _entries.Length; ++c)
{
_entries[c] = new SubIdvEntry.TransposedSubIdv(this, reader, c);
}
_schema = new SchemaImpl(this);
if (!HasRowData)
{
_colTransposers = new Transposer[_header.ColumnCount];
_colTransposersLock = new object();
}
}
}
public static ColumnInferenceResult InferColumns(MLContext context, IMultiStreamSource multiStreamSource,
string label, bool hasHeader, string separator, bool?isQuoted, bool?isSparse)
{
// heuristic: use first stream in multi-stream source to infer column types & split
var stream = multiStreamSource.Open(0);
var sample = TextFileSample.CreateFromFullStream(stream);
Func <TextLoader, IDataView> createDataView = (textLoader) =>
{
return(textLoader.Read(multiStreamSource));
};
return(InferColumns(context, sample, createDataView, label, hasHeader, separator, isQuoted, isSparse));
}
private static Stream OpenStream(IMultiStreamSource files)
{
Contracts.CheckValue(files, nameof(files));
Contracts.CheckParam(files.Count == 1, nameof(files), "Parquet loader must be created with one file");
return(files.Open(0));
}