public void Execute(ConsoleEnvironment env, string[] args)
{
if (args.Length > 0)
{
IConsoleCommand value;
if (!env.Commands.TryGetValue(args[0], out value))
{
env.log.Error("Can't find help for '{0}'", args[0]);
return;
}
env.log.Info(value.Usage ?? "No Help available");
return;
}
env.log.Info("Available commands");
foreach (var group in env.Commands.GroupBy(p => p.Value).OrderBy(c => c.GetType().Name))
{
var keys = string.Join(", ", group.Select(p => p.Key.ToUpperInvariant()));
env.log.Info(" {0}", keys);
var usage = @group.Key.Usage;
if (!string.IsNullOrWhiteSpace(usage))
{
env.log.Debug(" {0}", usage);
}
}
}
public void Execute(ConsoleEnvironment env, string[] args)
{
if (args.Length > 0)
{
var myId = 1;
int.TryParse(args[0], out myId);
}
var id = new DrillingDepartmentId(1);
var story = new List<Command>
{
// create some Command messages that are defined in Messages.cs
// and were auto-generated by lokad-codedsl from the text in Messages.ddd
// new OpenDrillingDepartment(id),
new ImportPlannedDepartmentStructureFromBid(id, new RequestId()),
};
// we have a story as a list of Commands, now send each one of the Commands to
// the Application Service that is hosting the DrillingDepartment Aggregate
// the Events generated by the DrillingDepartment Aggregate's that process the Commands will be
// output to the console in a human-readable way and can be read like a "story about a day in this department"
foreach (var command in story)
{
env.DrillingDepartmentAppService.Execute(command);
}
}
public void Execute(ConsoleEnvironment env, string[] args)
{
if (args.Length != 1)
throw new ArgumentException("Expected at least 1 args");
var id = int.Parse(args[0]);
env.DrillingDepartmentAppService.When(new CreateDrillingDepartmentPlan(new DrillingDepartmentId(id), new RequestId()));
env.log.Info("Drilling Department Plan Created!");
}
public void Execute(ConsoleEnvironment env, string[] args)
{
if (args.Length != 1)
env.log.Error("Expected at least 1 args");
var id = int.Parse(args[0]);
env.DrillingDepartmentAppService.When(new DrillShotPoints(new DrillingDepartmentId(id), new RequestId()));
env.log.Info("Shotpoints Drilled!");
}
public void Execute(ConsoleEnvironment env, string[] args)
{
if (args.Length != 1)
throw new ArgumentException("Expected at least 1 args");
var id = int.Parse(args[0]);
env.DrillingDepartmentAppService.When(new AssignTargetMetricsToDrillWorkunits(new DrillingDepartmentId(id), new RequestId()));
env.log.Info("Target Metrics Assigned!");
}
public void Execute(ConsoleEnvironment env, string[] args)
{
if (args.Length != 1)
throw new ArgumentException("Expected at least 1 args");
var id = int.Parse(args[0]);
env.DrillingDepartmentAppService.When(new CalculateStoreOmittedOrPartlyCompletedShotPoints(new DrillingDepartmentId(id), new RequestId()));
env.log.Info("Omitted Or Partly Completed ShotPoints Calculated!");
}
public void Execute(ConsoleEnvironment env, string[] args)
{
if (args.Length != 1)
throw new ArgumentException("Expected at least 1 args");
var id = int.Parse(args[0]);
env.DrillingDepartmentAppService.When(new IssueMapsLineTracesGpsCordinatesToDrillWorkunits(new DrillingDepartmentId(id), new RequestId()));
env.log.Info("Maps Issued to Drill Workunits!");
}
public void Execute(ConsoleEnvironment env, string[] args)
{
if (args.Length != 1)
throw new ArgumentException("Expected 1 args");
var id = int.Parse(args[0]);
env.DrillingDepartmentAppService.When(new SendMapsLineTracesGpsCordinatesToTheChiefDriller(new DrillingDepartmentId(id), new RequestId()));
env.log.Info("Maps Sent to Chief Driller!");
}
public void Execute(ConsoleEnvironment env, string[] args)
{
if (args.Length == 0)
{
env.Log.Error("You must specify action name");
return;
}
env.Tenant.When(new CaptureInboxEntry(env.Id, Guid.Empty, string.Join(" ", args)));
}
public void Execute(ConsoleEnvironment env, string[] args)
{
const int wginteger = 1000;
var requestId = new RequestId(Guid.NewGuid());
var workgroupName = string.Join(" ", args);
var we = new WorkgroupEntity(wginteger, requestId, workgroupName);
env.DrillingDepartmentAppService.When(new AddPlannedWorkgroupToDepartment(env.Session.SystemId, requestId, we));
env.log.Info("Workgroup added to Department!");
}
public void Execute(ConsoleEnvironment env, string[] args)
{
if (args.Length != 3)
throw new ArgumentException("Expected 3 args");
var id = int.Parse(args[0]);
// TODO - Fix lines below
// var employee = string.Join(" ", args.Skip(1));
// var employee = string.Join(" ", args.Skip(2));
env.DrillingDepartmentAppService.When(new SendSurveyedPointsToChiefDriller(new DrillingDepartmentId(id), new RequestId(), new DateTime()));
env.log.Info("Surveyed Points Issued to Drill Workunits!");
}
public void Execute(ConsoleEnvironment env, string[] args)
{
var entries = env.Session.GetCurrentSystem();
var workgroups = entries.Workgroups;
env.log.Info("Inbox ({0} Workgroups)", workgroups.Count);
foreach (var entry in workgroups)
{
var shortId = env.Session.MakePartialKey(entry.Key.Id);
env.log.Info(string.Format(" {0} {1, -20} {2, -20} {3, -20}", shortId, entry.Value.WorkgroupNumber, entry.Value.Name, entry.Value.Type));
}
}
public void Execute(ConsoleEnvironment env, string[] args)
{
if (args.Length != 2)
{
throw new ArgumentException("Expected 1 args");
}
var id = int.Parse(args[0]);
// TODO - Fix lines below
// var employee = string.Join(" ", args.Skip(1));
env.DrillingDepartmentAppService.When(new AmendActualDrillWorkunitResources(new DrillingDepartmentId(id), new RequestId()));
env.log.Info("Actual Drill Workunit Resources Amended!");
}
public void Execute(ConsoleEnvironment env, string[] args)
{
if (!env.Inbox.Inbox.ContainsKey(env.Id))
{
env.Log.Error("Tenant not defined");
return;
}
env.Log.Info("Inbox {0}", env.Id.Id);
foreach (var entry in env.Inbox.Inbox[env.Id].Items)
{
env.Log.Info(entry);
}
}
public void Execute(ConsoleEnvironment env, string[] args)
{
if (args.Length < 2)
{
throw new KnownConsoleInputError("At least 2 arguments are expected");
}
var item = env.Session.MatchItem(args[0]);
var newName = string.Join(" ", args.Skip(1));
var workgroupentity = item as WorkgroupEntityView;
if (workgroupentity != null)
{
env.DrillingDepartmentAppService.When(new ChangeWorkgroupName(env.Session.SystemId, workgroupentity.Id, newName));
env.log.Info("Workgroup name changed!");
return;
}
}
static void Main(string[] args)
{
var onnxModel = Path.Combine("Assets", "Model", "model.onnx");
var imagePathsData = Path.Combine("Assets", "Data", "images.tsv");
var imageFolder = Path.Combine("Assets", "Images");
using (var environment = new ConsoleEnvironment())
{
var imageHeight = 64;
var imageWidth = 64;
var loader = TextLoader.CreateReader(environment, context => (
ImagePath: context.LoadText(0),
Name: context.LoadText(1)),
separator: '\t',
hasHeader: false);
var data = loader.Read(new MultiFileSource(imagePathsData));
var estimator = loader.MakeNewEstimator()
.Append(row => (
Name: row.Name,
input: row.ImagePath.LoadAsImage(imageFolder).AsGrayscale().Resize(imageWidth, imageHeight).ExtractPixels()))
.Append(row => (row.Name, EmotionScores: row.input.ApplyOnnxModel(onnxModel)));
var model = estimator.Fit(data);
var predictionFunction = model.AsDynamic.MakePredictionFunction <EmotionData, EmotionPrediction>(environment);
var prediction = predictionFunction.Predict(new EmotionData()
{
ImagePath = "1.jpg"
});
int emotion = GetEmotion(prediction.PredictedLabels);
Console.WriteLine(GetEmotionString(emotion));
Console.ReadLine();
}
}
[ConditionalFact(typeof(Environment), nameof(Environment.Is64BitProcess))] // LightGBM is 64-bit only
public void MultiClassLightGBM()
{
var env = new ConsoleEnvironment(seed: 0);
var dataPath = GetDataPath(TestDatasets.iris.trainFilename);
var dataSource = new MultiFileSource(dataPath);
var ctx = new MulticlassClassificationContext(env);
var reader = TextLoader.CreateReader(env,
c => (label: c.LoadText(0), features: c.LoadFloat(1, 4)));
OvaPredictor pred = null;
// With a custom loss function we no longer get calibrated predictions.
var est = reader.MakeNewEstimator()
.Append(r => (label: r.label.ToKey(), r.features))
.Append(r => (r.label, preds: ctx.Trainers.LightGbm(
r.label,
r.features, onFit: p => pred = p)));
var pipe = reader.Append(est);
Assert.Null(pred);
var model = pipe.Fit(dataSource);
Assert.NotNull(pred);
var data = model.Read(dataSource);
// Just output some data on the schema for fun.
var schema = data.AsDynamic.Schema;
for (int c = 0; c < schema.ColumnCount; ++c)
{
Console.WriteLine($"{schema.GetColumnName(c)}, {schema.GetColumnType(c)}");
}
var metrics = ctx.Evaluate(data, r => r.label, r => r.preds, 2);
Assert.True(metrics.LogLoss > 0);
Assert.True(metrics.TopKAccuracy > 0);
}
public void SetupIrisPipeline()
{
_irisExample = new IrisData()
{
SepalLength = 3.3f,
SepalWidth = 1.6f,
PetalLength = 0.2f,
PetalWidth = 5.1f,
};
string _irisDataPath = Program.GetInvariantCultureDataPath("iris.txt");
using (var env = new ConsoleEnvironment(seed: 1, conc: 1, verbose: false, sensitivity: MessageSensitivity.None, outWriter: EmptyWriter.Instance))
{
var reader = new TextLoader(env,
new TextLoader.Arguments()
{
Separator = "\t",
HasHeader = true,
Column = new[]
{
new TextLoader.Column("Label", DataKind.R4, 0),
new TextLoader.Column("SepalLength", DataKind.R4, 1),
new TextLoader.Column("SepalWidth", DataKind.R4, 2),
new TextLoader.Column("PetalLength", DataKind.R4, 3),
new TextLoader.Column("PetalWidth", DataKind.R4, 4),
}
});
IDataView data = reader.Read(new MultiFileSource(_irisDataPath));
var pipeline = new ConcatEstimator(env, "Features", new[] { "SepalLength", "SepalWidth", "PetalLength", "PetalWidth" })
.Append(new SdcaMultiClassTrainer(env, new SdcaMultiClassTrainer.Arguments {
NumThreads = 1, ConvergenceTolerance = 1e-2f
}, "Features", "Label"));
var model = pipeline.Fit(data);
_irisModel = model.MakePredictionFunction <IrisData, IrisPrediction>(env);
}
}
[ConditionalFact(typeof(Environment), nameof(Environment.Is64BitProcess))] // LightGBM is 64-bit only
public void LightGbmBinaryClassification()
{
var env = new ConsoleEnvironment(seed: 0);
var dataPath = GetDataPath(TestDatasets.breastCancer.trainFilename);
var dataSource = new MultiFileSource(dataPath);
var ctx = new BinaryClassificationContext(env);
var reader = TextLoader.CreateReader(env,
c => (label: c.LoadBool(0), features: c.LoadFloat(1, 9)));
IPredictorWithFeatureWeights <float> pred = null;
var est = reader.MakeNewEstimator()
.Append(r => (r.label, preds: ctx.Trainers.LightGbm(r.label, r.features,
numBoostRound: 10,
numLeaves: 5,
learningRate: 0.01,
onFit: (p) => { pred = p; })));
var pipe = reader.Append(est);
Assert.Null(pred);
var model = pipe.Fit(dataSource);
Assert.NotNull(pred);
// 9 input features, so we ought to have 9 weights.
VBuffer <float> weights = new VBuffer <float>();
pred.GetFeatureWeights(ref weights);
Assert.Equal(9, weights.Length);
var data = model.Read(dataSource);
var metrics = ctx.Evaluate(data, r => r.label, r => r.preds);
// Run a sanity check against a few of the metrics.
Assert.InRange(metrics.Accuracy, 0, 1);
Assert.InRange(metrics.Auc, 0, 1);
Assert.InRange(metrics.Auprc, 0, 1);
}
[ConditionalFact(typeof(Environment), nameof(Environment.Is64BitProcess))] // LightGBM is 64-bit only
public void LightGBMRanking()
{
var env = new ConsoleEnvironment(seed: 0);
var dataPath = GetDataPath(TestDatasets.adultRanking.trainFilename);
var dataSource = new MultiFileSource(dataPath);
var ctx = new RankingContext(env);
var reader = TextLoader.CreateReader(env,
c => (label: c.LoadFloat(0), features: c.LoadFloat(9, 14), groupId: c.LoadText(1)),
separator: '\t', hasHeader: true);
LightGbmRankingPredictor pred = null;
var est = reader.MakeNewEstimator()
.Append(r => (r.label, r.features, groupId: r.groupId.ToKey()))
.Append(r => (r.label, r.groupId, score: ctx.Trainers.LightGbm(r.label, r.features, r.groupId, onFit: (p) => { pred = p; })));
var pipe = reader.Append(est);
Assert.Null(pred);
var model = pipe.Fit(dataSource);
Assert.NotNull(pred);
var data = model.Read(dataSource);
var metrics = ctx.Evaluate(data, r => r.label, r => r.groupId, r => r.score);
Assert.NotNull(metrics);
Assert.True(metrics.Ndcg.Length == metrics.Dcg.Length && metrics.Dcg.Length == 3);
Assert.InRange(metrics.Dcg[0], 1.4, 1.6);
Assert.InRange(metrics.Dcg[1], 1.4, 1.8);
Assert.InRange(metrics.Dcg[2], 1.4, 1.8);
Assert.InRange(metrics.Ndcg[0], 36.5, 37);
Assert.InRange(metrics.Ndcg[1], 36.5, 37);
Assert.InRange(metrics.Ndcg[2], 36.5, 37);
}
public void PrivateGetSetProperties()
{
var data = new List <ClassWithGetter>()
{
new ClassWithGetter(),
new ClassWithGetter(),
new ClassWithGetter()
};
using (var env = new ConsoleEnvironment())
{
var dataView = ComponentCreation.CreateDataView(env, data);
var enumeratorSimple = dataView.AsEnumerable <ClassWithSetter>(env, false).GetEnumerator();
var originalEnumerator = data.GetEnumerator();
while (enumeratorSimple.MoveNext() && originalEnumerator.MoveNext())
{
Assert.True(enumeratorSimple.Current.GetDay == originalEnumerator.Current.Day &&
enumeratorSimple.Current.GetHour == originalEnumerator.Current.Hour);
}
}
}
public void Pkpd()
{
var dataPath = GetDataPath(IrisDataPath);
using (var env = new ConsoleEnvironment())
{
var calibrator = new PavCalibratorTrainer(env);
var data = new TextLoader(env, GetIrisLoaderArgs())
.Read(new MultiFileSource(dataPath));
var sdcaTrainer = new LinearClassificationTrainer(env, new LinearClassificationTrainer.Arguments {
MaxIterations = 100, Shuffle = true, NumThreads = 1
}, "Features", "Label");
var pipeline = new TermEstimator(env, "Label")
.Append(new Pkpd(env, sdcaTrainer))
.Append(new KeyToValueEstimator(env, "PredictedLabel"));
TestEstimatorCore(pipeline, data);
}
}
public void TestWhiteningOldSavingAndLoading()
{
var env = new ConsoleEnvironment(seed: 0);
string dataSource = GetDataPath("generated_regression_dataset.csv");
var dataView = TextLoader.CreateReader(env,
c => (label: c.LoadFloat(11), features: c.LoadFloat(0, 10)),
separator: ';', hasHeader: true)
.Read(dataSource).AsDynamic;
var pipe = new VectorWhiteningEstimator(env, "features", "whitened");
var result = pipe.Fit(dataView).Transform(dataView);
var resultRoles = new RoleMappedData(result);
using (var ms = new MemoryStream())
{
TrainUtils.SaveModel(Env, Env.Start("saving"), ms, null, resultRoles);
ms.Position = 0;
var loadedView = ModelFileUtils.LoadTransforms(Env, dataView, ms);
}
Done();
}
public void Execute(ConsoleEnvironment env, string[] args)
{
if (args.Length != 0)
{
throw new ArgumentException("Expected 0 args");
}
var entries = env.Session.GetCurrentSystem();
var workgroups = entries.Workgroups;
foreach (var entry in workgroups)
{
// if (entry.Value.Structure == null) return;
// TODO - ADD MISSING COMMAND HERE
// e.g. env.DrillingDepartmentAppService.When(new AddWorkgroupToPlannedStructure(env.Session.SystemId, requestId, we));
env.log.Info("The Drilling Department has a Planned Department Structure");
env.log.Info(" ");
}
}
public void SimpleImageSmokeTest()
{
var env = new ConsoleEnvironment(0, verbose: true);
var reader = TextLoader.CreateReader(env,
ctx => ctx.LoadText(0).LoadAsImage().AsGrayscale().Resize(10, 8).ExtractPixels());
var schema = reader.AsDynamic.GetOutputSchema();
Assert.True(schema.TryGetColumnIndex("Data", out int col), "Could not find 'Data' column");
var type = schema.GetColumnType(col);
Assert.True(type.IsKnownSizeVector, $"Type was supposed to be known size vector but was instead '{type}'");
var vecType = type.AsVector;
Assert.Equal(NumberType.R4, vecType.ItemType);
Assert.Equal(3, vecType.DimCount);
Assert.Equal(3, vecType.GetDim(0));
Assert.Equal(8, vecType.GetDim(1));
Assert.Equal(10, vecType.GetDim(2));
}
public void OnnxModelMultiInput()
{
var modelFile = Path.Combine(Directory.GetCurrentDirectory(), "twoinput", "twoinput.onnx");
var env = new ConsoleEnvironment(seed: 1);
var samplevector = GetSampleArrayData();
var dataView = ML.Data.LoadFromEnumerable(
new TestDataMulti[] {
new TestDataMulti()
{
ina = new float[] { 1, 2, 3, 4, 5 },
inb = new float[] { 1, 2, 3, 4, 5 }
}
});
var pipeline = ML.Transforms.ApplyOnnxModel(new[] { "outa", "outb" }, new[] { "ina", "inb" }, modelFile);
var onnxTransformer = pipeline.Fit(dataView);
var onnx = onnxTransformer.Transform(dataView);
var outaCol = onnx.Schema["outa"];
var outbCol = onnx.Schema["outb"];
using (var curs = onnx.GetRowCursor(outaCol, onnx.Schema["outb"]))
{
var getScoresa = curs.GetGetter <VBuffer <float> >(outaCol);
var getScoresb = curs.GetGetter <VBuffer <float> >(outbCol);
var buffera = default(VBuffer <float>);
var bufferb = default(VBuffer <float>);
while (curs.MoveNext())
{
getScoresa(ref buffera);
getScoresb(ref bufferb);
Assert.Equal(5, buffera.Length);
Assert.Equal(5, bufferb.Length);
Assert.Equal(0, buffera.GetValues().ToArray().Sum());
Assert.Equal(30, bufferb.GetValues().ToArray().Sum());
}
}
(onnxTransformer as IDisposable)?.Dispose();
}
[ConditionalFact(typeof(Environment), nameof(Environment.Is64BitProcess))] // TensorFlow is 64-bit only
public void TestTensorFlowEntryPoint()
{
var dataPath = GetDataPath("Train-Tiny-28x28.txt");
using (var env = new ConsoleEnvironment(42))
{
var experiment = env.CreateExperiment();
var importInput = new Legacy.Data.TextLoader(dataPath);
importInput.Arguments.Column = new TextLoaderColumn[]
{
new TextLoaderColumn {
Name = "Label", Source = new[] { new TextLoaderRange(0) }
},
new TextLoaderColumn {
Name = "Placeholder", Source = new[] { new TextLoaderRange(1, 784) }
}
};
var importOutput = experiment.Add(importInput);
var tfTransformInput = new Legacy.Transforms.TensorFlowScorer
{
Data = importOutput.Data,
ModelLocation = "mnist_model/frozen_saved_model.pb",
InputColumns = new[] { "Placeholder" },
OutputColumns = new[] { "Softmax" },
};
var tfTransformOutput = experiment.Add(tfTransformInput);
experiment.Compile();
experiment.SetInput(importInput.InputFile, new SimpleFileHandle(env, dataPath, false, false));
experiment.Run();
var data = experiment.GetOutput(tfTransformOutput.OutputData);
var schema = data.Schema;
Assert.Equal(3, schema.ColumnCount);
Assert.Equal("Softmax", schema.GetColumnName(2));
Assert.Equal(10, schema.GetColumnType(2).VectorSize);
}
}
public void Train(string dest)
{
using (var env = new ConsoleEnvironment(verbose: false))
{
var args = new TextLoader.Arguments()
{
Separator = ",",
HasHeader = true,
Column = new TextLoader.Column[] {
new TextLoader.Column("Label", DataKind.R4, 0),
new TextLoader.Column("Sepal_length", DataKind.R4, 1),
new TextLoader.Column("Sepal_width", DataKind.R4, 2),
new TextLoader.Column("Petal_length", DataKind.R4, 3),
new TextLoader.Column("Petal_width", DataKind.R4, 4),
}
};
var reader = new TextLoader(env, args);
var concat = new ColumnConcatenatingEstimator(env,
"Features", "Sepal_length",
"Sepal_width", "Petal_length", "Petal_width");
var km = new MulticlassLogisticRegression(env, "Label", "Features");
var pipeline = concat.Append(km);
IDataView trainingDataView = reader.Read(new MultiFileSource(_dataset));
var model = pipeline.Fit(trainingDataView);
var obs = new IrisObservation()
{
Sepal_length = 3.3f,
Sepal_width = 1.6f,
Petal_length = 0.2f,
Petal_width = 5.1f,
};
_fct = model.MakePredictionFunction <IrisObservation, IrisPrediction>(env);
using (var stdest = File.OpenWrite(dest))
model.SaveTo(env, stdest);
}
}
public void TestCustomTransformer()
{
string dataPath = GetDataPath("adult.test");
var source = new MultiFileSource(dataPath);
var loader = ML.Data.TextReader(new[] {
new TextLoader.Column("Float1", DataKind.R4, 0),
new TextLoader.Column("Float4", DataKind.R4, new[] { new TextLoader.Range(0), new TextLoader.Range(2), new TextLoader.Range(4), new TextLoader.Range(10) })
}, s => { s.Separator = ","; s.HasHeader = true; });
var data = loader.Read(source);
IDataView transformedData;
// We create a temporary environment to instantiate the custom transformer. This is to ensure that we don't need the same
// environment for saving and loading.
using (var tempoEnv = new ConsoleEnvironment())
{
var customEst = new CustomMappingEstimator <MyInput, MyOutput>(tempoEnv, MyLambda.MyAction, "MyLambda");
try
{
TestEstimatorCore(customEst, data);
Assert.True(false, "Cannot work without MEF injection");
}
catch (Exception)
{
// REVIEW: we should have a common mechanism that will make sure this is 'our' exception thrown.
}
ML.CompositionContainer = new CompositionContainer(new TypeCatalog(typeof(MyLambda)));
TestEstimatorCore(customEst, data);
transformedData = customEst.Fit(data).Transform(data);
}
var inputs = transformedData.AsEnumerable <MyInput>(ML, true);
var outputs = transformedData.AsEnumerable <MyOutput>(ML, true);
Assert.True(inputs.Zip(outputs, (x, y) => y.Together == $"{x.Float1} + {string.Join(", ", x.Float4)}").All(x => x));
Done();
}
/// <summary>
/// Reads a text file as a IDataView.
/// Follows pandas API.
/// </summary>
/// <param name="filename">filename</param>
/// <param name="sep">column separator</param>
/// <param name="header">has a header or not</param>
/// <param name="names">column names (can be empty)</param>
/// <param name="dtypes">column types (can be empty)</param>
/// <param name="nrows">number of rows to read</param>
/// <param name="guess_rows">number of rows used to guess types</param>
/// <param name="encoding">text encoding</param>
/// <param name="useThreads">specific to TextLoader</param>
/// <param name="host">host</param>
/// <param name="index">add a column to hold the index</param>
/// <returns>TextLoader</returns>
public static IDataView ReadCsvToTextLoader(string[] filenames,
char sep = ',', bool header = true,
string[] names = null, DataViewType[] dtypes = null,
int nrows = -1, int guess_rows = 10,
Encoding encoding = null, bool useThreads = true,
bool index = false, IHost host = null)
{
var df = ReadCsv(filenames[0], sep: sep, header: header, names: names, dtypes: dtypes,
nrows: guess_rows, guess_rows: guess_rows, encoding: encoding, index: index);
var sch = df.Schema;
var cols = new TextLoader.Column[sch.Count];
for (int i = 0; i < cols.Length; ++i)
{
cols[i] = TextLoader.Column.Parse(df.NameType(i));
if (cols[i] == null)
{
throw Contracts.Except("Unable to parse '{0}'.", df.NameType(i));
}
}
var args = new TextLoader.Options()
{
AllowQuoting = false,
Separators = new[] { sep },
Columns = cols,
TrimWhitespace = true,
UseThreads = useThreads,
HasHeader = header,
MaxRows = nrows > 0 ? (int?)nrows : null
};
if (host == null)
{
host = new ConsoleEnvironment().Register("TextLoader");
}
var multiSource = new MultiFileSource(filenames);
return(new TextLoader(host, args, multiSource).Load(multiSource));
}
/// <summary>
/// Process a script to be parsed (from the input resource).
/// </summary>
private static void Process(IndentingTextWriter wrt, string text, ArgsBase defaults)
{
var env = new ConsoleEnvironment(seed: 42);
using (wrt.Nest())
{
var args1 = defaults.Clone();
using (wrt.Nest())
{
if (!CmdParser.ParseArguments(env, text, args1, s => wrt.WriteLine("*** {0}", s)))
{
wrt.WriteLine("*** Failed!");
}
}
string str1 = args1.ToString();
wrt.WriteLine("ToString: {0}", str1);
string settings1 = CmdParser.GetSettings(env, args1, defaults, SettingsFlags.None);
wrt.WriteLine("Settings: {0}", settings1);
var args2 = defaults.Clone();
using (wrt.Nest())
{
if (!CmdParser.ParseArguments(env, settings1, args2, s => wrt.WriteLine("*** BUG: {0}", s)))
{
wrt.WriteLine("*** BUG: parsing result of GetSettings failed!");
}
}
string str2 = args2.ToString();
if (str1 != str2)
{
wrt.WriteLine("*** BUG: ToString Mismatch: {0}", str2);
}
string settings2 = CmdParser.GetSettings(env, args2, defaults, SettingsFlags.None);
if (settings1 != settings2)
{
wrt.WriteLine("*** BUG: Settings Mismatch: {0}", settings2);
}
}
}
public void PrincipalComponentAnalysis()
{
var env = new ConsoleEnvironment(seed: 0);
var dataPath = GetDataPath("generated_regression_dataset.csv");
var dataSource = new MultiFileSource(dataPath);
var reader = TextLoader.CreateReader(env,
c => (label: c.LoadFloat(11), features: c.LoadFloat(0, 10)),
separator: ';', hasHeader: true);
var data = reader.Read(dataSource);
var est = reader.MakeNewEstimator()
.Append(r => (r.label,
pca: r.features.ToPrincipalComponents(rank: 5)));
var tdata = est.Fit(data).Transform(data);
var schema = tdata.AsDynamic.Schema;
Assert.True(schema.TryGetColumnIndex("pca", out int pcaCol));
var type = schema.GetColumnType(pcaCol);
Assert.True(type.IsVector && type.IsKnownSizeVector && type.ItemType.IsNumber);
}
public void TestTextDatasetLearn()
{
using (var env = new ConsoleEnvironment())
{
string pathData = GetDataPath(@"../UnitTest/tweets_labeled_10k_test_validation.tsv");
int batchSize = 5;
int numIterations = 35;
int numTransformLevels = 1;
int numSampleRows = 100;
SupportedMetric metric = PipelineSweeperSupportedMetrics.GetSupportedMetric(PipelineSweeperSupportedMetrics.Metrics.AccuracyMicro);
// Using the simple, uniform random sampling (with replacement) engine
PipelineOptimizerBase autoMlEngine = new UniformRandomEngine(env);
// Test initial learning
var amls = AutoInference.InferPipelines(env, autoMlEngine, pathData, "", out var _, numTransformLevels, batchSize,
metric, out var _, numSampleRows, new IterationTerminator(numIterations),
MacroUtils.TrainerKinds.SignatureMultiClassClassifierTrainer);
env.Check(amls.GetAllEvaluatedPipelines().Length == numIterations);
}
Done();
}
public void OnnxModelMultiInput()
{
var modelFile = Path.Combine(Directory.GetCurrentDirectory(), "twoinput", "twoinput.onnx");
using (var env = new ConsoleEnvironment(seed: 1, conc: 1))
{
var samplevector = GetSampleArrayData();
var dataView = ML.Data.ReadFromEnumerable(
new TestDataMulti[] {
new TestDataMulti()
{
ina = new float[] { 1, 2, 3, 4, 5 },
inb = new float[] { 1, 2, 3, 4, 5 }
}
});
var onnx = ML.Transforms.ApplyOnnxModel(modelFile, new[] { "outa", "outb" }, new[] { "ina", "inb" }).Fit(dataView).Transform(dataView);
onnx.Schema.TryGetColumnIndex("outa", out int scoresa);
onnx.Schema.TryGetColumnIndex("outb", out int scoresb);
using (var curs = onnx.GetRowCursor(onnx.Schema["outa"], onnx.Schema["outb"]))
{
var getScoresa = curs.GetGetter <VBuffer <float> >(scoresa);
var getScoresb = curs.GetGetter <VBuffer <float> >(scoresb);
var buffera = default(VBuffer <float>);
var bufferb = default(VBuffer <float>);
while (curs.MoveNext())
{
getScoresa(ref buffera);
getScoresb(ref bufferb);
Assert.Equal(5, buffera.Length);
Assert.Equal(5, bufferb.Length);
Assert.Equal(0, buffera.GetValues().ToArray().Sum());
Assert.Equal(30, bufferb.GetValues().ToArray().Sum());
}
}
}
}
public void PcaWorkout()
{
var env = new ConsoleEnvironment(seed: 1, conc: 1);
string dataSource = GetDataPath("generated_regression_dataset.csv");
var data = TextLoader.CreateReader(env,
c => (label: c.LoadFloat(11), features: c.LoadFloat(0, 10)),
separator: ';', hasHeader: true)
.Read(new MultiFileSource(dataSource));
var invalidData = TextLoader.CreateReader(env,
c => (label: c.LoadFloat(11), features: c.LoadText(0, 10)),
separator: ';', hasHeader: true)
.Read(new MultiFileSource(dataSource));
var est = new PcaEstimator(env, "features", "pca", rank: 5, advancedSettings: s => {
s.Seed = 1;
});
// The following call fails because of the following issue
// https://github.com/dotnet/machinelearning/issues/969
// TestEstimatorCore(est, data.AsDynamic, invalidInput: invalidData.AsDynamic);
var outputPath = GetOutputPath("PCA", "pca.tsv");
using (var ch = env.Start("save"))
{
var saver = new TextSaver(env, new TextSaver.Arguments {
Silent = true, OutputHeader = false
});
IDataView savedData = TakeFilter.Create(env, est.Fit(data.AsDynamic).Transform(data.AsDynamic), 4);
savedData = new ChooseColumnsTransform(env, savedData, "pca");
using (var fs = File.Create(outputPath))
DataSaverUtils.SaveDataView(ch, saver, savedData, fs, keepHidden: true);
}
CheckEquality("PCA", "pca.tsv");
Done();
}
public void LambdaTransformCreate()
{
using (var env = new ConsoleEnvironment(42))
{
var data = ReadBreastCancerExamples();
var idv = env.CreateDataView(data);
var filter = LambdaTransform.CreateFilter <BreastCancerExample, object>(env, idv,
(input, state) => input.Label == 0, null);
Assert.Null(filter.GetRowCount());
// test re-apply
var applied = env.CreateDataView(data);
applied = ApplyTransformUtils.ApplyAllTransformsToData(env, filter, applied);
var saver = new TextSaver(env, new TextSaver.Arguments());
Assert.True(applied.Schema.TryGetColumnIndex("Label", out int label));
using (var fs = File.Create(GetOutputPath(OutputRelativePath, "lambda-output.tsv")))
saver.SaveData(fs, applied, label);
}
}
private static (IEnumerable <string> epListContents, JObject manifest) BuildManifests()
{
ConsoleEnvironment env = new ConsoleEnvironment();
foreach (Type type in _types)
{
env.ComponentCatalog.RegisterAssembly(type.Assembly);
}
var catalog = env.ComponentCatalog;
var regex = new Regex(@"\r\n?|\n", RegexOptions.Compiled);
var epListContents = catalog.AllEntryPoints()
.Select(x => string.Join("\t",
x.Name,
regex.Replace(x.Description, ""),
x.Method.DeclaringType,
x.Method.Name,
x.InputType,
x.OutputType)
.Replace(Environment.NewLine, "", StringComparison.Ordinal))
.OrderBy(x => x);
var manifest = JsonManifestUtils.BuildAllManifests(env, catalog);
//clean up the description from the new line characters
if (manifest[FieldNames.TopEntryPoints] != null && manifest[FieldNames.TopEntryPoints] is JArray)
{
foreach (JToken entry in manifest[FieldNames.TopEntryPoints].Children())
{
if (entry[FieldNames.Desc] != null)
{
entry[FieldNames.Desc] = regex.Replace(entry[FieldNames.Desc].ToString(), "");
}
}
}
return(epListContents, manifest);
}
public void OVAWithAllConstructorArgs()
{
var dataPath = GetDataPath(IrisDataPath);
string featNam = "Features";
string labNam = "Label";
using (var env = new ConsoleEnvironment())
{
var calibrator = new FixedPlattCalibratorTrainer(env, new FixedPlattCalibratorTrainer.Arguments());
var data = new TextLoader(env, GetIrisLoaderArgs()).Read(new MultiFileSource(dataPath));
var averagePerceptron = new AveragedPerceptronTrainer(env, new AveragedPerceptronTrainer.Arguments {
FeatureColumn = featNam, LabelColumn = labNam, Shuffle = true, Calibrator = null
});
var pipeline = new TermEstimator(env, labNam)
.Append(new Ova(env, averagePerceptron, labNam, true, calibrator: calibrator, 10000, true))
.Append(new KeyToValueEstimator(env, "PredictedLabel"));
TestEstimatorCore(pipeline, data);
}
}
public IRowCursor[] GetRowCursorSet(out IRowCursorConsolidator consolidator, Func <int, bool> predicate, int n, IRandom rand = null)
{
var host = new ConsoleEnvironment().Register("Estimate n threads");
n = DataViewUtils.GetThreadCount(host, n);
if (n <= 1)
{
consolidator = null;
return(new IRowCursor[] { GetRowCursor(predicate, rand) });
}
else
{
var cursors = Source.GetRowCursorSet(out consolidator, i => predicate(i) || predicate(SchemaHelper.NeedColumn(_columnMapping, i)),
n, rand);
for (int i = 0; i < cursors.Length; ++i)
{
cursors[i] = new AddRandomCursor(this, cursors[i]);
}
return(cursors);
}
}
/// <summary>
/// Creates a new environment. It should be done
/// with <tt>using</tt>.
/// </summary>
public static ConsoleEnvironment NewTestEnvironment(int?seed = null, bool verbose = false,
MessageSensitivity sensitivity = (MessageSensitivity)(-1),
int conc = 0, TextWriter outWriter = null, TextWriter errWriter = null)
{
if (!seed.HasValue)
{
seed = 42;
}
if (outWriter == null)
{
outWriter = new StreamWriter(new MemoryStream());
}
if (errWriter == null)
{
errWriter = new StreamWriter(new MemoryStream());
}
var env = new ConsoleEnvironment(seed, verbose, sensitivity, conc, outWriter, errWriter);
ComponentHelper.AddStandardComponents(env);
return(env);
}
public void ConcatWith()
{
var env = new ConsoleEnvironment(seed: 0);
var dataPath = GetDataPath("iris.data");
var reader = TextLoader.CreateReader(env,
c => (label: c.LoadText(4), values: c.LoadFloat(0, 3), value: c.LoadFloat(2)),
separator: ',');
var dataSource = new MultiFileSource(dataPath);
var data = reader.Read(dataSource);
var est = data.MakeNewEstimator()
.Append(r => (
r.label, r.values, r.value,
c0: r.label.AsVector(), c1: r.label.ConcatWith(r.label),
c2: r.value.ConcatWith(r.values), c3: r.values.ConcatWith(r.value, r.values)));
var tdata = est.Fit(data).Transform(data);
var schema = tdata.AsDynamic.Schema;
int[] idx = new int[4];
for (int i = 0; i < idx.Length; ++i)
{
Assert.True(schema.TryGetColumnIndex("c" + i, out idx[i]), $"Could not find col c{i}");
}
var types = new VectorType[idx.Length];
int[] expectedLen = new int[] { 1, 2, 5, 9 };
for (int i = 0; i < idx.Length; ++i)
{
var type = schema.GetColumnType(idx[i]);
Assert.True(type.VectorSize > 0, $"Col c{i} had unexpected type {type}");
types[i] = type.AsVector;
Assert.Equal(expectedLen[i], type.VectorSize);
}
Assert.Equal(TextType.Instance, types[0].ItemType);
Assert.Equal(TextType.Instance, types[1].ItemType);
Assert.Equal(NumberType.Float, types[2].ItemType);
Assert.Equal(NumberType.Float, types[3].ItemType);
}
public void AveragePerceptronNoCalibration()
{
var env = new ConsoleEnvironment(seed: 0);
var dataPath = GetDataPath(TestDatasets.breastCancer.trainFilename);
var dataSource = new MultiFileSource(dataPath);
var ctx = new BinaryClassificationContext(env);
var reader = TextLoader.CreateReader(env,
c => (label: c.LoadBool(0), features: c.LoadFloat(1, 9)));
LinearBinaryPredictor pred = null;
var loss = new HingeLoss(new HingeLoss.Arguments()
{
Margin = 1
});
var est = reader.MakeNewEstimator()
.Append(r => (r.label, preds: ctx.Trainers.AveragedPerceptron(r.label, r.features, lossFunction: loss,
numIterations: 2, onFit: p => pred = p)));
var pipe = reader.Append(est);
Assert.Null(pred);
var model = pipe.Fit(dataSource);
Assert.NotNull(pred);
// 9 input features, so we ought to have 9 weights.
Assert.Equal(9, pred.Weights2.Count);
var data = model.Read(dataSource);
var metrics = ctx.Evaluate(data, r => r.label, r => r.preds);
// Run a sanity check against a few of the metrics.
Assert.InRange(metrics.Accuracy, 0, 1);
Assert.InRange(metrics.Auc, 0, 1);
Assert.InRange(metrics.Auprc, 0, 1);
}
public void ToKey()
{
var env = new ConsoleEnvironment(seed: 0);
var dataPath = GetDataPath("iris.data");
var reader = TextLoader.CreateReader(env,
c => (label: c.LoadText(4), values: c.LoadFloat(0, 3)),
separator: ',');
var dataSource = new MultiFileSource(dataPath);
var data = reader.Read(dataSource);
var est = data.MakeNewEstimator()
.Append(r => (labelKey: r.label.ToKey(), valuesKey: r.values.ToKey(onFit: m => { })))
.Append(r => (r.labelKey, r.valuesKey, valuesKeyKey: r.valuesKey.ToKey()));
var tdata = est.Fit(data).Transform(data);
var schema = tdata.AsDynamic.Schema;
Assert.True(schema.TryGetColumnIndex("labelKey", out int labelCol));
Assert.True(schema.TryGetColumnIndex("valuesKey", out int valuesCol));
Assert.True(schema.TryGetColumnIndex("valuesKeyKey", out int valuesKeyCol));
Assert.Equal(3, schema.GetColumnType(labelCol).KeyCount);
Assert.True(schema.GetColumnType(valuesCol).ItemType.IsKey);
Assert.True(schema.GetColumnType(valuesKeyCol).ItemType.IsKey);
var labelKeyType = schema.GetMetadataTypeOrNull(MetadataUtils.Kinds.KeyValues, labelCol);
var valuesKeyType = schema.GetMetadataTypeOrNull(MetadataUtils.Kinds.KeyValues, valuesCol);
var valuesKeyKeyType = schema.GetMetadataTypeOrNull(MetadataUtils.Kinds.KeyValues, valuesKeyCol);
Assert.NotNull(labelKeyType);
Assert.NotNull(valuesKeyType);
Assert.NotNull(valuesKeyKeyType);
Assert.True(labelKeyType.IsVector && labelKeyType.ItemType == TextType.Instance);
Assert.True(valuesKeyType.IsVector && valuesKeyType.ItemType == NumberType.Float);
Assert.True(valuesKeyKeyType.IsVector && valuesKeyKeyType.ItemType == NumberType.Float);
// Because they're over exactly the same data, they ought to have the same cardinality and everything.
Assert.True(valuesKeyKeyType.Equals(valuesKeyType));
}
[ConditionalFact(typeof(Environment), nameof(Environment.Is64BitProcess))] // x86 output differs from Baseline
public void OnnxModelScenario()
{
if (!RuntimeInformation.IsOSPlatform(OSPlatform.Windows))
{
return;
}
var modelFile = "squeezenet/00000001/model.onnx";
using (var env = new ConsoleEnvironment(seed: 1, conc: 1))
{
var samplevector = GetSampleArrayData();
var dataView = ComponentCreation.CreateDataView(Env,
new TestData[] {
new TestData()
{
data_0 = samplevector
}
});
var onnx = OnnxTransform.Create(env, dataView, modelFile,
new[] { "data_0" },
new[] { "softmaxout_1" });
onnx.Schema.TryGetColumnIndex("softmaxout_1", out int scores);
using (var curs = onnx.GetRowCursor(col => col == scores))
{
var getScores = curs.GetGetter <VBuffer <float> >(scores);
var buffer = default(VBuffer <float>);
while (curs.MoveNext())
{
getScores(ref buffer);
Assert.Equal(1000, buffer.Length);
}
}
}
}
private void GenericDenseDataView <T>(T[] v1, T[] v2)
{
var inputs = new[] {
new DenseExample <T>()
{
X = v1
},
new DenseExample <T>()
{
X = v2
}
};
using (var host = new ConsoleEnvironment())
{
var data = host.CreateStreamingDataView(inputs);
var value = new VBuffer <T>();
int n = 0;
using (var cur = data.GetRowCursor(i => true))
{
var getter = cur.GetGetter <VBuffer <T> >(0);
while (cur.MoveNext())
{
getter(ref value);
Assert.True(value.GetValues().Length == 3);
++n;
}
}
Assert.True(n == 2);
var iter = data.AsEnumerable <DenseExample <T> >(host, false).GetEnumerator();
n = 0;
while (iter.MoveNext())
{
++n;
}
Assert.True(n == 2);
}
}
public void TestEstimatorSaveLoad()
{
using (var env = new ConsoleEnvironment())
{
var dataFile = GetDataPath("images/images.tsv");
var imageFolder = Path.GetDirectoryName(dataFile);
var data = TextLoader.Create(env, new TextLoader.Arguments()
{
Column = new[]
{
new TextLoader.Column("ImagePath", DataKind.TX, 0),
new TextLoader.Column("Name", DataKind.TX, 1),
}
}, new MultiFileSource(dataFile));
var pipe = new ImageLoaderEstimator(env, imageFolder, ("ImagePath", "ImageReal"))
.Append(new ImageResizerEstimator(env, "ImageReal", "ImageReal", 100, 100))
.Append(new ImagePixelExtractorEstimator(env, "ImageReal", "ImagePixels"))
.Append(new ImageGrayscaleEstimator(env, ("ImageReal", "ImageGray")));
pipe.GetOutputSchema(Core.Data.SchemaShape.Create(data.Schema));
var model = pipe.Fit(data);
using (var file = env.CreateTempFile())
{
using (var fs = file.CreateWriteStream())
model.SaveTo(env, fs);
var model2 = TransformerChain.LoadFrom(env, file.OpenReadStream());
var newCols = ((ImageLoaderTransform)model2.First()).Columns;
var oldCols = ((ImageLoaderTransform)model.First()).Columns;
Assert.True(newCols
.Zip(oldCols, (x, y) => x == y)
.All(x => x));
}
}
Done();
}
public void Execute(ConsoleEnvironment env, string[] args)
{
if (args.Length > 0)
{
IConsoleCommand value;
if (!env.Commands.TryGetValue(args[0], out value))
{
env.Log.Error("Can't find help for '{0}'", args[0]);
return;
}
env.Log.Info(value.Usage ?? "No Help available");
return;
}
env.Log.Info("Available commands");
foreach (var actionHandler in env.Commands.OrderBy(h => h.Key))
{
env.Log.Info(" {0}", actionHandler.Key.ToUpperInvariant());
if (!string.IsNullOrWhiteSpace(actionHandler.Value.Usage))
{
env.Log.Info(" {0}", actionHandler.Value.Usage);
}
}
}
public void Execute(ConsoleEnvironment env, string[] args)
{
Environment.Exit(0);
}
