public void GetModelWithDataView()
{
var textLoader = AutoMlModelCreation.MlContextInstance
.Data.CreateTextLoader <ChordData>(separatorChar: ',', hasHeader: true);
var trainData = textLoader.Load("./Resources/trainData.csv", "./Resources/testData.csv");
var(experimentResult, predictionEngine) =
AutoMlModelCreation.CreateModelGivenDataView(trainData, 1);
Assert.IsNotNull(experimentResult);
var metrics =
AutoMlModelCreation.EvaluateModel(experimentResult, "./Resources/testData.csv");
Assert.IsTrue(metrics.LogLoss < 0.5);
var prediction =
predictionEngine.Predict(AutoMlModelCreation.GetChordDataFromPcp(emPcp));
Assert.IsTrue(prediction.ChordPrediction.ToLower().Equals("em"));
var dPrediction =
predictionEngine.Predict(AutoMlModelCreation.GetChordDataFromPcp(dPcp));
Assert.IsTrue(dPrediction.ChordPrediction.ToLower().Equals("d"));
}
private static MulticlassClassificationMetrics GetAndPrintValidationMetricsForData(
ExperimentResult <MulticlassClassificationMetrics> experimentResult, string fileName)
{
Console.WriteLine($@"Running experiment for dataset {fileName}");
var validationMetrics =
AutoMlModelCreation.EvaluateModel(experimentResult,
fileName);
Console.WriteLine(@"Experiment ran with the following results");
Console.WriteLine(
$@"LogLoss={validationMetrics.LogLoss} the closer to 0 the better");
Console.WriteLine($@"Confusion Matrix Actuals\Predicted");
var confusionMatrix = validationMetrics.ConfusionMatrix;
for (var i = 0; i < confusionMatrix.NumberOfClasses; i++)
{
for (var j = 0; j < confusionMatrix.NumberOfClasses; j++)
{
Console.Write(confusionMatrix.Counts[i][j] + "\t");
}
Console.WriteLine();
}
return(validationMetrics);
}
public string GetPrediction(float[] sample, int sampleRate)
{
var pcp =
Profiling.Profiling.PitchClassProfileForSamples(sample,
sampleRate);
var chordData = AutoMlModelCreation.GetChordDataFromPcp(pcp);
var prediction = engine.Predict(chordData).ChordPrediction;
return(System.Globalization.CultureInfo.CurrentCulture.TextInfo
.ToTitleCase(prediction.ToLower()));
}
static async Task Main(string[] args)
{
uint secondsToRun = 1;
string directory = null;
if (args.Length >= 1)
{
secondsToRun = uint.Parse(args[0]);
}
if (args.Length >= 2)
{
directory = args[1];
}
if (directory != null)
{
var trainDataGenerator = new FileSystemTrainDataGenerator(directory, Path.Combine(directory, "trainData.csv"));
await trainDataGenerator.GenerateTrainData();
}
var currentTime = DateTimeOffset.UtcNow.ToUnixTimeSeconds();
Console.WriteLine(
$@"Training Prediction model with AutoML for {secondsToRun} seconds");
var textLoader = AutoMlModelCreation.MlContextInstance
.Data.CreateTextLoader <ChordData>(separatorChar: ',', hasHeader: true);
var trainDataFiles = Directory.GetFiles(directory, "*.csv");
var trainData = textLoader.Load(trainDataFiles.Append("./Resources/trainData.csv").ToArray());
var(_, modelWithLabelMapping, experimentResult) =
AutoMlModelCreation.CreateTransformerGivenDataView(trainData, secondsToRun);
Console.WriteLine();
GetAndPrintValidationMetricsForData(experimentResult, "./Resources/testData.csv");
Console.WriteLine();
var validationMetrics = GetAndPrintValidationMetricsForData(experimentResult, "./Resources/trainData.csv");
Console.WriteLine();
AutoMlModelCreation.MlContextInstance.Model.Save(
modelWithLabelMapping, trainData.Schema,
$@"./generatedModels/model{currentTime}S{secondsToRun}L{validationMetrics.LogLoss}.model");
}
public void GetModel_TrainsTheModelCorrectly()
{
var(experimentResult, predictionEngine) =
AutoMlModelCreation.CreateModel("./Resources/trainData.csv", 1);
Assert.IsNotNull(experimentResult);
var metrics =
AutoMlModelCreation.EvaluateModel(experimentResult, "./Resources/testData.csv");
Assert.IsTrue(metrics.LogLoss < 0.5);
var prediction =
predictionEngine.Predict(AutoMlModelCreation.GetChordDataFromPcp(emPcp));
Assert.IsTrue(prediction.ChordPrediction.ToLower().Equals("em"));
var dPrediction =
predictionEngine.Predict(AutoMlModelCreation.GetChordDataFromPcp(dPcp));
Assert.IsTrue(dPrediction.ChordPrediction.ToLower().Equals("d"));
}
public async Task CreateModelGivenInitialDataAndStoredChordsFolder()
{
const string trainDataFile = "./Resources/trainData.csv";
const string testDataFile = "./Resources/testData.csv";
const string inputDirectory = "./Resources/trainDataGeneratorFolderTraining/";
const uint timeoutInSeconds = 1;
const string outputDirectory = "./Resources/generatedModels/";
try
{
Directory.CreateDirectory(inputDirectory);
Directory.CreateDirectory(outputDirectory);
Assert.AreEqual(Directory.GetFiles(outputDirectory, "*.model").Length, 0);
var respository = new FileSystemChordRepository(inputDirectory);
var chords = new Chord[10];
for (var i = 0; i < 10; i++)
{
chords[i] = ChordTest.ChordExample();
}
foreach (Chord chord in chords)
{
respository.SaveChord(chord);
}
var(experimentResult, predictionEngine) =
await AutoMlModelCreation.CreateModelGivenInitialDataAndStoredChordsFolder(
trainDataFile, testDataFile, inputDirectory, timeoutInSeconds, outputDirectory, new Progress <(int, string)>());
Assert.IsNotNull(experimentResult);
var metrics =
AutoMlModelCreation.EvaluateModel(experimentResult, "./Resources/testData.csv");
Assert.IsTrue(metrics.LogLoss < 0.5);
var prediction =
predictionEngine.Predict(AutoMlModelCreation.GetChordDataFromPcp(emPcp));
Assert.IsTrue(prediction.ChordPrediction.ToLower().Equals("em"));
var dPrediction =
predictionEngine.Predict(AutoMlModelCreation.GetChordDataFromPcp(dPcp));
Assert.IsTrue(dPrediction.ChordPrediction.ToLower().Equals("d"));
var isModelPresent = Directory.GetFiles(outputDirectory, "*.model").Length == 1;
Assert.IsTrue(isModelPresent);
} finally
{
File.Delete(Path.Combine(inputDirectory, "trainData.csv"));
var filesToDelete = Directory.GetFiles(inputDirectory, "*.json");
foreach (string fileToDelete in filesToDelete)
{
File.Delete(fileToDelete);
}
Directory.Delete(inputDirectory);
var modelsToDelete = Directory.GetFiles(outputDirectory, "*.model");
foreach (string modelToDelete in modelsToDelete)
{
File.Delete(modelToDelete);
}
Directory.Delete(outputDirectory);
}
}
