// Note that we don't filter out rows with parsing issues since it's not acceptable to
// produce a different set of rows when subsetting columns. Any parsing errors need to be
// translated to NaN, not result in skipping the row. We should produce some diagnostics
// to alert the user to the issues.
private Cursor(TextLoader parent, ParseStats stats, bool[] active, LineReader reader, int srcNeeded, int cthd)
: base(parent._host)
{
Ch.Assert(active == null || active.Length == parent._bindings.Infos.Length);
Ch.AssertValue(reader);
Ch.AssertValue(stats);
Ch.Assert(srcNeeded >= 0);
Ch.Assert(cthd > 0);
_total = -1;
_batch = -1;
_bindings = parent._bindings;
_parser = parent._parser;
_active = active;
_reader = reader;
_stats = stats;
_srcNeeded = srcNeeded;
ParallelState state = null;
if (cthd > 1)
{
state = new ParallelState(this, out _rows, cthd);
}
else
{
_rows = _parser.CreateRowSet(_stats, 1, _active);
}
try
{
_getters = new Delegate[_bindings.Infos.Length];
for (int i = 0; i < _getters.Length; i++)
{
if (_active != null && !_active[i])
{
continue;
}
ColumnPipe v = _rows.Pipes[i];
Ch.Assert(v != null);
_getters[i] = v.GetGetter();
Ch.Assert(_getters[i] != null);
}
if (state != null)
{
_ator = ParseParallel(state).GetEnumerator();
state = null;
}
else
{
_ator = ParseSequential().GetEnumerator();
}
}
finally
{
if (state != null)
{
state.Dispose();
}
}
}
// This method is called if only a datafile is specified, without a loader/term and value columns.
// It determines the type of the Value column and returns the appropriate TextLoader component factory.
private static IComponentFactory <IMultiStreamSource, IDataLoader> GetLoaderFactory(string filename, bool keyValues, IHost host)
{
Contracts.AssertValue(host);
// If the user specified non-key values, we define the value column to be numeric.
if (!keyValues)
{
return(ComponentFactoryUtils.CreateFromFunction <IMultiStreamSource, IDataLoader>(
(env, files) => TextLoader.Create(
env,
new TextLoader.Arguments()
{
Column = new[]
{
new TextLoader.Column("Term", DataKind.TX, 0),
new TextLoader.Column("Value", DataKind.Num, 1)
}
},
files)));
}
// If the user specified key values, we scan the values to determine the range of the key type.
ulong min = ulong.MaxValue;
ulong max = ulong.MinValue;
try
{
var txtArgs = new TextLoader.Arguments();
bool parsed = CmdParser.ParseArguments(host, "col=Term:TX:0 col=Value:TX:1", txtArgs);
host.Assert(parsed);
var data = TextLoader.ReadFile(host, txtArgs, new MultiFileSource(filename));
using (var cursor = data.GetRowCursor(c => true))
{
var getTerm = cursor.GetGetter <ReadOnlyMemory <char> >(0);
var getVal = cursor.GetGetter <ReadOnlyMemory <char> >(1);
ReadOnlyMemory <char> txt = default;
using (var ch = host.Start("Creating Text Lookup Loader"))
{
long countNonKeys = 0;
while (cursor.MoveNext())
{
getVal(ref txt);
ulong res;
// Try to parse the text as a key value between 1 and ulong.MaxValue. If this succeeds and res>0,
// we update max and min accordingly. If res==0 it means the value is missing, in which case we ignore it for
// computing max and min.
if (Conversions.Instance.TryParseKey(ref txt, 1, ulong.MaxValue, out res))
{
if (res < min && res != 0)
{
min = res;
}
if (res > max)
{
max = res;
}
}
// If parsing as key did not succeed, the value can still be 0, so we try parsing it as a ulong. If it succeeds,
// then the value is 0, and we update min accordingly.
else if (Conversions.Instance.TryParse(ref txt, out res))
{
ch.Assert(res == 0);
min = 0;
}
//If parsing as a ulong fails, we increment the counter for the non-key values.
else
{
var term = default(ReadOnlyMemory <char>);
getTerm(ref term);
if (countNonKeys < 5)
{
ch.Warning("Term '{0}' in mapping file is mapped to non key value '{1}'", term, txt);
}
countNonKeys++;
}
}
if (countNonKeys > 0)
{
ch.Warning("Found {0} non key values in the file '{1}'", countNonKeys, filename);
}
if (min > max)
{
min = 0;
max = uint.MaxValue - 1;
ch.Warning("did not find any valid key values in the file '{0}'", filename);
}
else
{
ch.Info("Found key values in the range {0} to {1} in the file '{2}'", min, max, filename);
}
ch.Done();
}
}
}
catch (Exception e)
{
throw host.Except(e, "Failed to parse the lookup file '{0}' in TermLookupTransform", filename);
}
TextLoader.Column valueColumn = new TextLoader.Column("Value", DataKind.U4, 1);
if (max - min < (ulong)int.MaxValue)
{
valueColumn.KeyRange = new KeyRange(min, max);
}
else if (max - min < (ulong)uint.MaxValue)
{
valueColumn.KeyRange = new KeyRange(min);
}
else
{
valueColumn.Type = DataKind.U8;
valueColumn.KeyRange = new KeyRange(min);
}
return(ComponentFactoryUtils.CreateFromFunction <IMultiStreamSource, IDataLoader>(
(env, files) => TextLoader.Create(
env,
new TextLoader.Arguments()
{
Column = new[]
{
new TextLoader.Column("Term", DataKind.TX, 0),
valueColumn
}
},
files)));
}
/// <summary>
/// Utility method to create the file-based <see cref="TermMap"/> if the <see cref="ArgumentsBase.DataFile"/>
/// argument of <paramref name="args"/> was present.
/// </summary>
private static TermMap CreateFileTermMap(IHostEnvironment env, IChannel ch, ArgumentsBase args, Builder bldr)
{
Contracts.AssertValue(ch);
ch.AssertValue(env);
ch.AssertValue(args);
ch.Assert(!string.IsNullOrWhiteSpace(args.DataFile));
ch.AssertValue(bldr);
string file = args.DataFile;
// First column using the file.
string src = args.TermsColumn;
IMultiStreamSource fileSource = new MultiFileSource(file);
var loaderFactory = args.Loader;
// If the user manually specifies a loader, or this is already a pre-processed binary
// file, then we assume the user knows what they're doing and do not attempt to convert
// to the desired type ourselves.
bool autoConvert = false;
IDataView termData;
if (loaderFactory != null)
{
termData = loaderFactory.CreateComponent(env, fileSource);
}
else
{
// Determine the default loader from the extension.
var ext = Path.GetExtension(file);
bool isBinary = string.Equals(ext, ".idv", StringComparison.OrdinalIgnoreCase);
bool isTranspose = string.Equals(ext, ".tdv", StringComparison.OrdinalIgnoreCase);
if (isBinary || isTranspose)
{
ch.Assert(isBinary != isTranspose);
ch.CheckUserArg(!string.IsNullOrWhiteSpace(src), nameof(args.TermsColumn),
"Must be specified");
if (isBinary)
{
termData = new BinaryLoader(env, new BinaryLoader.Arguments(), fileSource);
}
else
{
ch.Assert(isTranspose);
termData = new TransposeLoader(env, new TransposeLoader.Arguments(), fileSource);
}
}
else
{
if (!string.IsNullOrWhiteSpace(src))
{
ch.Warning(
"{0} should not be specified when default loader is TextLoader. Ignoring {0}={1}",
nameof(Arguments.TermsColumn), src);
}
termData = TextLoader.ReadFile(env,
new TextLoader.Arguments()
{
Separator = "tab",
Column = new[] { new TextLoader.Column("Term", DataKind.TX, 0) }
},
fileSource);
src = "Term";
autoConvert = true;
}
}
ch.AssertNonEmpty(src);
int colSrc;
if (!termData.Schema.TryGetColumnIndex(src, out colSrc))
{
throw ch.ExceptUserArg(nameof(args.TermsColumn), "Unknown column '{0}'", src);
}
var typeSrc = termData.Schema.GetColumnType(colSrc);
if (!autoConvert && !typeSrc.Equals(bldr.ItemType))
{
throw ch.ExceptUserArg(nameof(args.TermsColumn), "Must be of type '{0}' but was '{1}'", bldr.ItemType, typeSrc);
}
using (var cursor = termData.GetRowCursor(col => col == colSrc))
using (var pch = env.StartProgressChannel("Building term dictionary from file"))
{
var header = new ProgressHeader(new[] { "Total Terms" }, new[] { "examples" });
var trainer = Trainer.Create(cursor, colSrc, autoConvert, int.MaxValue, bldr);
double rowCount = termData.GetRowCount(true) ?? double.NaN;
long rowCur = 0;
pch.SetHeader(header,
e =>
{
e.SetProgress(0, rowCur, rowCount);
// Purely feedback for the user. That the other thread might be
// working in the background is not a problem.
e.SetMetric(0, trainer.Count);
});
while (cursor.MoveNext() && trainer.ProcessRow())
{
rowCur++;
}
if (trainer.Count == 0)
{
ch.Warning("Term map loaded from file resulted in an empty map.");
}
pch.Checkpoint(trainer.Count, rowCur);
return(trainer.Finish());
}
}
// This method is called if only a datafile is specified, without a loader/term and value columns.
// It determines the type of the Value column and returns the appropriate TextLoader component factory.
private static IComponentFactory <IMultiStreamSource, IDataLoader> GetLoaderFactory(string filename, bool keyValues, IHost host)
{
Contracts.AssertValue(host);
// If the user specified non-key values, we define the value column to be numeric.
if (!keyValues)
{
return(ComponentFactoryUtils.CreateFromFunction <IMultiStreamSource, IDataLoader>(
(env, files) => TextLoader.Create(
env,
new TextLoader.Arguments()
{
Column = new[]
{
new TextLoader.Column("Term", DataKind.TX, 0),
new TextLoader.Column("Value", DataKind.Num, 1)
}
},
files)));
}
// If the user specified key values, we scan the values to determine the range of the key type.
ulong min = ulong.MaxValue;
ulong max = ulong.MinValue;
try
{
var txtArgs = new TextLoader.Arguments();
bool parsed = CmdParser.ParseArguments(host, "col=Term:TX:0 col=Value:TX:1", txtArgs);
host.Assert(parsed);
var data = TextLoader.ReadFile(host, txtArgs, new MultiFileSource(filename));
using (var cursor = data.GetRowCursor(c => true))
{
var getTerm = cursor.GetGetter <ReadOnlyMemory <char> >(0);
var getVal = cursor.GetGetter <ReadOnlyMemory <char> >(1);
ReadOnlyMemory <char> txt = default;
using (var ch = host.Start("Creating Text Lookup Loader"))
{
long countNonKeys = 0;
while (cursor.MoveNext())
{
getVal(ref txt);
ulong res;
// Try to parse the text as a key value between 1 and ulong.MaxValue. If this succeeds and res>0,
// we update max and min accordingly. If res==0 it means the value is missing, in which case we ignore it for
// computing max and min.
if (Conversions.Instance.TryParseKey(in txt, 1, ulong.MaxValue, out res))
{
if (res < min && res != 0)
{
min = res;
}
if (res > max)
{
max = res;
}
}
// If parsing as key did not succeed, the value can still be 0, so we try parsing it as a ulong. If it succeeds,
// then the value is 0, and we update min accordingly.
else if (Conversions.Instance.TryParse(in txt, out res))
{
ch.Assert(res == 0);
min = 0;
}
// This method is called if only a datafile is specified, without a loader/term and value columns.
// It determines the type of the Value column and returns the appropriate TextLoader subcomponent.
private static SubComponent <IDataLoader, SignatureDataLoader> GetLoaderSubComponent(string filename, bool keyValues, IHost host)
{
Contracts.AssertValue(host);
// If the user specified non-key values, we define the value column to be numeric.
if (!keyValues)
{
return(new SubComponent <IDataLoader, SignatureDataLoader>("Text", "col=Term:TX:0", "col=Value:Num:1"));
}
// If the user specified key values, we scan the values to determine the range of the key type.
ulong min = ulong.MaxValue;
ulong max = ulong.MinValue;
try
{
var txtArgs = new TextLoader.Arguments();
bool parsed = CmdParser.ParseArguments(host, "col=Term:TX:0 col=Value:TX:1", txtArgs);
host.Assert(parsed);
var txtLoader = new TextLoader(host, txtArgs, new MultiFileSource(filename));
using (var cursor = txtLoader.GetRowCursor(c => true))
{
var getTerm = cursor.GetGetter <DvText>(0);
var getVal = cursor.GetGetter <DvText>(1);
DvText txt = default(DvText);
using (var ch = host.Start("Creating Text Lookup Loader"))
{
long countNonKeys = 0;
while (cursor.MoveNext())
{
getVal(ref txt);
ulong res;
// Try to parse the text as a key value between 1 and ulong.MaxValue. If this succeeds and res>0,
// we update max and min accordingly. If res==0 it means the value is missing, in which case we ignore it for
// computing max and min.
if (Conversions.Instance.TryParseKey(ref txt, 1, ulong.MaxValue, out res))
{
if (res < min && res != 0)
{
min = res;
}
if (res > max)
{
max = res;
}
}
// If parsing as key did not succeed, the value can still be 0, so we try parsing it as a ulong. If it succeeds,
// then the value is 0, and we update min accordingly.
else if (Conversions.Instance.TryParse(ref txt, out res))
{
ch.Assert(res == 0);
min = 0;
}
//If parsing as a ulong fails, we increment the counter for the non-key values.
else
{
var term = default(DvText);
getTerm(ref term);
if (countNonKeys < 5)
{
ch.Warning("Term '{0}' in mapping file is mapped to non key value '{1}'", term, txt);
}
countNonKeys++;
}
}
if (countNonKeys > 0)
{
ch.Warning("Found {0} non key values in the file '{1}'", countNonKeys, filename);
}
if (min > max)
{
min = 0;
max = uint.MaxValue - 1;
ch.Warning("did not find any valid key values in the file '{0}'", filename);
}
else
{
ch.Info("Found key values in the range {0} to {1} in the file '{2}'", min, max, filename);
}
ch.Done();
}
}
}
catch (Exception e)
{
throw host.Except(e, "Failed to parse the lookup file '{0}' in TermLookupTransform", filename);
}
string settings;
if (max - min < (ulong)int.MaxValue)
{
settings = string.Format("col=Value:U4[{0}-{1}]:1", min, max);
}
else if (max - min < (ulong)uint.MaxValue)
{
settings = string.Format("col=Value:U4[{0}-*]:1", min);
}
else
{
settings = string.Format("col=Value:U8[{0}-*]:1", min);
}
return(new SubComponent <IDataLoader, SignatureDataLoader>("Text", "col=Term:TXT:0", settings));
}