public TransposeLoader(IHostEnvironment env, Arguments args, IMultiStreamSource file) { Contracts.CheckValue(env, nameof(env)); _host = env.Register(LoadName); _host.CheckValue(args, nameof(args)); _host.CheckValue(file, nameof(file)); _host.Check(file.Count == 1, "Transposed loader accepts a single file only"); _threads = args.Threads ?? 0; if (_threads < 0) { _threads = 0; } _file = file; using (Stream stream = _file.Open(0)) using (BinaryReader reader = new BinaryReader(stream)) { _header = InitHeader(reader); reader.Seek(_header.SubIdvTableOffset); _schemaEntry = new SubIdvEntry.SchemaSubIdv(this, reader); _entries = new SubIdvEntry.TransposedSubIdv[_header.ColumnCount]; for (int c = 0; c < _entries.Length; ++c) { _entries[c] = new SubIdvEntry.TransposedSubIdv(this, reader, c); } _schema = new SchemaImpl(this); if (!HasRowData) { _colTransposers = new Transposer[_header.ColumnCount]; _colTransposersLock = new object(); } } }
private TransposeLoader(IHost host, ModelLoadContext ctx, IMultiStreamSource file) { Contracts.CheckValue(host, nameof(host)); _host = host; _host.CheckValue(file, nameof(file)); _host.Check(file.Count == 1, "Transposed loader accepts a single file only"); // *** Binary format ** // int: Number of threads if explicitly defined, or 0 if the // number of threads was automatically determined _threads = ctx.Reader.ReadInt32(); _host.CheckDecode(_threads >= 0); // Dedupe code somehow? _file = file; using (Stream stream = _file.Open(0)) using (BinaryReader reader = new BinaryReader(stream)) { _header = InitHeader(reader); reader.Seek(_header.SubIdvTableOffset); _schemaEntry = new SubIdvEntry.SchemaSubIdv(this, reader); _entries = new SubIdvEntry.TransposedSubIdv[_header.ColumnCount]; for (int c = 0; c < _entries.Length; ++c) { _entries[c] = new SubIdvEntry.TransposedSubIdv(this, reader, c); } _schema = new SchemaImpl(this); if (!HasRowData) { _colTransposers = new Transposer[_header.ColumnCount]; _colTransposersLock = new object(); } } }
public static ColumnInferenceResult InferColumns(MLContext context, IMultiStreamSource multiStreamSource, string label, bool hasHeader, string separator, bool?isQuoted, bool?isSparse) { // heuristic: use first stream in multi-stream source to infer column types & split var stream = multiStreamSource.Open(0); var sample = TextFileSample.CreateFromFullStream(stream); Func <TextLoader, IDataView> createDataView = (textLoader) => { return(textLoader.Read(multiStreamSource)); }; return(InferColumns(context, sample, createDataView, label, hasHeader, separator, isQuoted, isSparse)); }
private static Stream OpenStream(IMultiStreamSource files) { Contracts.CheckValue(files, nameof(files)); Contracts.CheckParam(files.Count == 1, nameof(files), "Parquet loader must be created with one file"); return(files.Open(0)); }