public void NormalizerWorkout()
{
string dataPath = GetDataPath("iris.txt");
var loader = new TextLoader(Env, new TextLoader.Arguments
{
Column = new[] {
new TextLoader.Column("float1", DataKind.R4, 1),
new TextLoader.Column("float4", DataKind.R4, new[] { new TextLoader.Range(1, 4) }),
new TextLoader.Column("double1", DataKind.R8, 1),
new TextLoader.Column("double4", DataKind.R8, new[] { new TextLoader.Range(1, 4) }),
new TextLoader.Column("int1", DataKind.I4, 0),
new TextLoader.Column("float0", DataKind.R4, new[] { new TextLoader.Range {
Min = 1, VariableEnd = true
} }),
},
HasHeader = true
}, new MultiFileSource(dataPath));
var est = new Normalizer(Env,
new Normalizer.MinMaxColumn("float1"),
new Normalizer.MinMaxColumn("float4"),
new Normalizer.MinMaxColumn("double1"),
new Normalizer.MinMaxColumn("double4"),
new Normalizer.BinningColumn("float1", "float1bin"),
new Normalizer.BinningColumn("float4", "float4bin"),
new Normalizer.BinningColumn("double1", "double1bin"),
new Normalizer.BinningColumn("double4", "double4bin"),
new Normalizer.MeanVarColumn("float1", "float1mv"),
new Normalizer.MeanVarColumn("float4", "float4mv"),
new Normalizer.MeanVarColumn("double1", "double1mv"),
new Normalizer.MeanVarColumn("double4", "double4mv"),
new Normalizer.LogMeanVarColumn("float1", "float1lmv"),
new Normalizer.LogMeanVarColumn("float4", "float4lmv"),
new Normalizer.LogMeanVarColumn("double1", "double1lmv"),
new Normalizer.LogMeanVarColumn("double4", "double4lmv"));
var data = loader.Read(new MultiFileSource(dataPath));
var badData1 = new CopyColumnsTransform(Env, ("int1", "float1")).Transform(data);
var badData2 = new CopyColumnsTransform(Env, ("float0", "float4")).Transform(data);
TestEstimatorCore(est, data, null, badData1);
TestEstimatorCore(est, data, null, badData2);
var outputPath = GetOutputPath("Normalizer", "normalized.tsv");
using (var ch = Env.Start("save"))
{
var saver = new TextSaver(Env, new TextSaver.Arguments {
Silent = true
});
using (var fs = File.Create(outputPath))
DataSaverUtils.SaveDataView(ch, saver, new DropColumnsTransform(Env, est.Fit(data).Transform(data), "float0"), fs, keepHidden: true);
}
CheckEquality("Normalizer", "normalized.tsv");
Done();
}
public void KeyToValueWorkout()
{
string dataPath = GetDataPath("iris.txt");
var reader = new TextLoader(Env, new TextLoader.Arguments
{
Column = new[]
{
new TextLoader.Column("ScalarString", DataKind.TX, 1),
new TextLoader.Column("VectorString", DataKind.TX, new[] { new TextLoader.Range(1, 4) }),
new TextLoader.Column
{
Name = "BareKey",
Source = new[] { new TextLoader.Range(0) },
Type = DataKind.U4,
KeyRange = new KeyRange(0, 5),
}
}
});
var data = reader.Read(dataPath);
data = new ValueToKeyMappingEstimator(Env, new[] {
new TermTransform.ColumnInfo("ScalarString", "A"),
new TermTransform.ColumnInfo("VectorString", "B")
}).Fit(data).Transform(data);
var badData1 = new CopyColumnsTransform(Env, ("BareKey", "A")).Transform(data);
var badData2 = new CopyColumnsTransform(Env, ("VectorString", "B")).Transform(data);
var est = new KeyToValueEstimator(Env, ("A", "A_back"), ("B", "B_back"));
TestEstimatorCore(est, data, invalidInput: badData1);
TestEstimatorCore(est, data, invalidInput: badData2);
var outputPath = GetOutputPath("KeyToValue", "featurized.tsv");
using (var ch = Env.Start("save"))
{
var saver = new TextSaver(Env, new TextSaver.Arguments {
Silent = true
});
IDataView savedData = est.Fit(data).Transform(data);
using (var fs = File.Create(outputPath))
DataSaverUtils.SaveDataView(ch, saver, savedData, fs, keepHidden: true);
}
CheckEquality("KeyToValue", "featurized.tsv");
Done();
}
/// <summary>
/// If any column names in <param name="colNames" /> are present in <param name="input" />, this
/// method will create a transform that copies them to temporary columns. It will populate <param name="hiddenNames" />
/// with an array of string pairs containing the original name and the generated temporary column name, respectively.
/// </summary>
/// <param name="env"></param>
private static IDataView AliasIfNeeded(IHostEnvironment env, IDataView input, string[] colNames, out KeyValuePair <string, string>[] hiddenNames)
{
hiddenNames = null;
var toHide = new List <string>(colNames.Length);
foreach (var name in colNames)
{
int discard;
if (input.Schema.TryGetColumnIndex(name, out discard))
{
toHide.Add(name);
}
}
if (toHide.Count == 0)
{
return(input);
}
hiddenNames = toHide.Select(colName =>
new KeyValuePair <string, string>(colName, input.Schema.GetTempColumnName(colName))).ToArray();
return(CopyColumnsTransform.Create(env, new CopyColumnsTransform.Arguments()
{
Column = hiddenNames.Select(pair => new CopyColumnsTransform.Column()
{
Name = pair.Value, Source = pair.Key
}).ToArray()
}, input));
}
public static CommonOutputs.TransformOutput RenameBinaryPredictionScoreColumns(IHostEnvironment env,
RenameBinaryPredictionScoreColumnsInput input)
{
Contracts.CheckValue(env, nameof(env));
var host = env.Register("ScoreModel");
host.CheckValue(input, nameof(input));
EntryPointUtils.CheckInputArgs(host, input);
if (input.PredictorModel.Predictor.PredictionKind == PredictionKind.BinaryClassification)
{
ColumnType labelType;
var labelNames = input.PredictorModel.GetLabelInfo(host, out labelType);
if (labelNames != null && labelNames.Length == 2)
{
var positiveClass = labelNames[1];
// Rename all the score columns.
int colMax;
var maxScoreId = input.Data.Schema.GetMaxMetadataKind(out colMax, MetadataUtils.Kinds.ScoreColumnSetId);
var copyCols = new List <(string Source, string Name)>();
for (int i = 0; i < input.Data.Schema.ColumnCount; i++)
{
if (input.Data.Schema.IsHidden(i))
{
continue;
}
if (!ShouldAddColumn(input.Data.Schema, i, null, maxScoreId))
{
continue;
}
// Do not rename the PredictedLabel column.
ReadOnlyMemory <char> tmp = default;
if (input.Data.Schema.TryGetMetadata(TextType.Instance, MetadataUtils.Kinds.ScoreValueKind, i,
ref tmp) &&
ReadOnlyMemoryUtils.EqualsStr(MetadataUtils.Const.ScoreValueKind.PredictedLabel, tmp))
{
continue;
}
var source = input.Data.Schema.GetColumnName(i);
var name = source + "." + positiveClass;
copyCols.Add((source, name));
}
var copyColumn = new CopyColumnsTransform(env, copyCols.ToArray()).Transform(input.Data);
var dropColumn = SelectColumnsTransform.CreateDrop(env, copyColumn, copyCols.Select(c => c.Source).ToArray());
return(new CommonOutputs.TransformOutput {
Model = new TransformModel(env, dropColumn, input.Data), OutputData = dropColumn
});
}
}
var newView = NopTransform.CreateIfNeeded(env, input.Data);
return(new CommonOutputs.TransformOutput {
Model = new TransformModel(env, newView, input.Data), OutputData = newView
});
}
public static CommonOutputs.TransformOutput CopyColumns(IHostEnvironment env, CopyColumnsTransform.Arguments input)
{
Contracts.CheckValue(env, nameof(env));
var host = env.Register("CopyColumns");
host.CheckValue(input, nameof(input));
EntryPointUtils.CheckInputArgs(host, input);
var xf = CopyColumnsTransform.Create(env, input, input.Data);
return(new CommonOutputs.TransformOutput {
Model = new TransformModel(env, xf, input.Data), OutputData = xf
});
}
private static IDataView UnaliasIfNeeded(IHostEnvironment env, IDataView input, KeyValuePair <string, string>[] hiddenNames)
{
if (Utils.Size(hiddenNames) == 0)
{
return(input);
}
input = CopyColumnsTransform.Create(env, new CopyColumnsTransform.Arguments()
{
Column = hiddenNames.Select(pair => new CopyColumnsTransform.Column()
{
Name = pair.Key, Source = pair.Value
}).ToArray()
}, input);
return(SelectColumnsTransform.CreateDrop(env, input, hiddenNames.Select(pair => pair.Value).ToArray()));
}
public static IDataTransform Create(IHostEnvironment env, Arguments args, IDataView input)
{
Contracts.CheckValue(env, nameof(env));
var h = env.Register(LoaderSignature);
h.CheckValue(args, nameof(args));
h.CheckValue(input, nameof(input));
h.CheckNonWhiteSpace(args.Source, nameof(args.Source));
if (string.IsNullOrWhiteSpace(args.Name))
{
args.Name = args.Source;
}
var file = Utils.FindExistentFileOrNull("pretrained.model", "Sentiment", assemblyForBasePath: typeof(SentimentAnalyzingTransform));
if (file == null)
{
throw h.Except("resourcePath", "Missing resource for SentimentAnalyzingTransform.");
}
// The logic below ensures that any columns in our input IDataView that conflict
// with column names known to be used in the pretrained model transform pipeline we're
// loading are aliased to temporary column names before we apply the pipeline and then
// renamed back to their original names after. We do this to ensure the pretrained model
// doesn't shadow or replace columns we aren't expecting it to.
// 1. Alias any column in the input IDataView that is known to appear to the pretrained
// model into a temporary column so that we can restore them after the pretrained model
// is added to the pipeline.
KeyValuePair <string, string>[] aliased;
input = AliasIfNeeded(env, input, _modelIntermediateColumnNames, out aliased);
// 2. Copy source column to a column with the name expected by the pretrained model featurization
// transform pipeline.
input = new CopyColumnsTransform(env, new CopyColumnsTransform.Arguments()
{
Column = new[] { new CopyColumnsTransform.Column()
{
Source = args.Source, Name = ModelInputColumnName
} }
}, input);
// 3. Apply the pretrained model and its featurization transform pipeline.
input = LoadTransforms(env, input, file);
// 4. Copy the output column from the pretrained model to a temporary column.
var scoreTempName = input.Schema.GetTempColumnName("sa_out");
input = new CopyColumnsTransform(env, new CopyColumnsTransform.Arguments()
{
Column = new [] { new CopyColumnsTransform.Column()
{
Name = scoreTempName, Source = ModelScoreColumnName
} }
}, input);
// 5. Drop all the columns created by the pretrained model, including the expected input column
// and the output column, which we have copied to a temporary column in (4).
input = new DropColumnsTransform(env, new DropColumnsTransform.Arguments()
{
Column = _modelIntermediateColumnNames
}, input);
// 6. Unalias all the original columns that were originally present in the IDataView, but may have
// been shadowed by column names in the pretrained model. This method will also drop all the temporary
// columns that were created for them in (1).
input = UnaliasIfNeeded(env, input, aliased);
// 7. Copy the temporary column with the score we created in (4) to a column with the user-specified destination name.
input = new CopyColumnsTransform(env, new CopyColumnsTransform.Arguments()
{
Column = new[] { new CopyColumnsTransform.Column()
{
Name = args.Name, Source = scoreTempName
} }
}, input);
// 8. Drop the temporary column with the score created in (4).
return(new DropColumnsTransform(env, new DropColumnsTransform.Arguments()
{
Column = new[] { scoreTempName }
}, input));
}
public void TensorFlowTransformMNISTConvTest()
{
var model_location = "mnist_model/frozen_saved_model.pb";
using (var env = new ConsoleEnvironment(seed: 1, conc: 1))
{
var dataPath = GetDataPath("Train-Tiny-28x28.txt");
var testDataPath = GetDataPath("MNIST.Test.tiny.txt");
// Pipeline
var loader = TextLoader.ReadFile(env,
new TextLoader.Arguments()
{
Separator = "tab",
HasHeader = true,
Column = new[]
{
new TextLoader.Column("Label", DataKind.Num, 0),
new TextLoader.Column("Placeholder", DataKind.Num, new [] { new TextLoader.Range(1, 784) })
}
}, new MultiFileSource(dataPath));
IDataView trans = CopyColumnsTransform.Create(env, new CopyColumnsTransform.Arguments()
{
Column = new[] { new CopyColumnsTransform.Column()
{
Name = "reshape_input", Source = "Placeholder"
} }
}, loader);
trans = TensorFlowTransform.Create(env, trans, model_location, new[] { "Softmax", "dense/Relu" }, new[] { "Placeholder", "reshape_input" });
trans = new ConcatTransform(env, "Features", "Softmax", "dense/Relu").Transform(trans);
var trainer = new LightGbmMulticlassTrainer(env, new LightGbmArguments());
var cached = new CacheDataView(env, trans, prefetch: null);
var trainRoles = new RoleMappedData(cached, label: "Label", feature: "Features");
var pred = trainer.Train(trainRoles);
// Get scorer and evaluate the predictions from test data
IDataScorerTransform testDataScorer = GetScorer(env, trans, pred, testDataPath);
var metrics = Evaluate(env, testDataScorer);
Assert.Equal(0.99, metrics.AccuracyMicro, 2);
Assert.Equal(1.0, metrics.AccuracyMacro, 2);
// Create prediction engine and test predictions
var model = env.CreatePredictionEngine <MNISTData, MNISTPrediction>(testDataScorer);
var sample1 = new MNISTData()
{
Placeholder = new float[] { 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 18, 18, 18, 126, 136, 175, 26,
166, 255, 247, 127, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 30, 36, 94, 154, 170, 253, 253, 253, 253, 253,
225, 172, 253, 242, 195, 64, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 49, 238, 253, 253, 253, 253, 253, 253, 253,
253, 251, 93, 82, 82, 56, 39, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 18, 219, 253, 253, 253, 253, 253, 198,
182, 247, 241, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 80, 156, 107, 253, 253, 205, 11, 0,
43, 154, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 14, 1, 154, 253, 90, 0, 0, 0, 0, 0, 0,
0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 139, 253, 190, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 11, 190, 253, 70, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 35, 241, 225, 160, 108, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 81, 240, 253, 253, 119, 25, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 45, 186, 253, 253, 150, 27, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 16, 93, 252, 253, 187, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 249, 253, 249, 64, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 46, 130, 183, 253, 253, 207, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 39, 148, 229, 253, 253, 253, 250, 182, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 24, 114, 221, 253, 253, 253, 253, 201, 78, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 23, 66, 213, 253, 253, 253, 253, 198, 81, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 18, 171, 219, 253, 253, 253, 253, 195, 80, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 55, 172, 226, 253, 253, 253, 253, 244, 133, 11, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 136, 253, 253, 253, 212, 135, 132, 16, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0 }
};
var prediction = model.Predict(sample1);
float max = -1;
int maxIndex = -1;
for (int i = 0; i < prediction.PredictedLabels.Length; i++)
{
if (prediction.PredictedLabels[i] > max)
{
max = prediction.PredictedLabels[i];
maxIndex = i;
}
}
Assert.Equal(5, maxIndex);
}
}
