public override bool TrainModel(TrainerCommandLineOptions options)
{
var fileName = FeatureExtractFolder(options);
var startDate = DateTime.Now;
var dataView = MlContext.Data.LoadFromTextFile <ClassificationData>(fileName, hasHeader: false);
var splitDataView = MlContext.Data.TrainTestSplit(dataView, testFraction: 0.2);
var featuresColumnName = "Features";
var estimator = MlContext.Transforms.Text.FeaturizeText(nameof(ClassificationData.NGramText))
.Append(MlContext.Transforms.NormalizeMeanVariance(nameof(ClassificationData.FileGroupType)))
.Append(MlContext.Transforms.Concatenate(featuresColumnName, nameof(ClassificationData.NGramText), nameof(ClassificationData.FileGroupType)))
.Append(MlContext.BinaryClassification.Trainers.FastTree(labelColumnName: "Label", featureColumnName: featuresColumnName));
var model = estimator.Fit(splitDataView.TrainSet);
Logger <TrainerCommandLineOptions> .Debug($"Model trained in {DateTime.Now.Subtract(startDate).TotalSeconds} seconds", options);
var predictions = model.Transform(splitDataView.TestSet);
var metrics = MlContext.BinaryClassification.Evaluate(predictions, "Label");
Logger <TrainerCommandLineOptions> .Debug($"Accuracy: {metrics.Accuracy} | F1: {metrics.F1Score} | Auc: {metrics.AreaUnderRocCurve}", options);
SaveModel(model, splitDataView.TrainSet.Schema, options);
return(true);
}
static void Main(string[] args)
{
#if RELEASE
var options = TrainerCommandLineParser.Parse(args);
if (options == null)
{
return;
}
#else
var options = new TrainerCommandLineOptions()
{
FolderOfData = args[0], LogLevel = LogLevels.DEBUG, ModelType = Enum.Parse <ModelType>(args[1])
};
#endif
try
{
switch (options.ModelType)
{
case ModelType.CLASSIFICATION:
new ClassificationEngine().TrainModel(options);
break;
case ModelType.CLUSTERING:
new ClusteringEngine().TrainModel(options);
break;
}
}
catch (Exception ex)
{
Console.WriteLine(ex);
}
}
protected void SaveModel(ITransformer trainedModel, DataViewSchema schema, TrainerCommandLineOptions options)
{
using (var fileStream = new FileStream(OutputModelPath, FileMode.Create, FileAccess.Write, FileShare.Write))
{
MlContext.Model.Save(trainedModel, schema, fileStream);
}
Logger <TrainerCommandLineOptions> .Debug($"Model saved to {OutputModelPath}", options);
}
public override bool TrainModel(TrainerCommandLineOptions options)
{
var fileName = FeatureExtractFolder(options);
var startDate = DateTime.Now;
var fullData = MlContext.Data.LoadFromTextFile(path: fileName,
new[]
{
new TextLoader.Column("Label", DataKind.Single, 0),
new TextLoader.Column(nameof(ClusterData.StartStringData), DataKind.String, 1),
new TextLoader.Column(nameof(ClusterData.EndStringData), DataKind.String, 2)
},
hasHeader: false,
separatorChar: ',');
var trainTestData = MlContext.Data.TrainTestSplit(fullData, testFraction: 0.2);
var trainingDataView = trainTestData.TrainSet;
var testingDataView = trainTestData.TestSet;
var featuresColumnName = "Features";
var pipeline = MlContext.Transforms.Text.NormalizeText(nameof(ClusterData.StartStringData))
.Append(MlContext.Transforms.Text.TokenizeIntoWords(nameof(ClusterData.StartStringData)))
.Append(MlContext.Transforms.Text.RemoveDefaultStopWords(nameof(ClusterData.StartStringData)))
.Append(MlContext.Transforms.Conversion.MapValueToKey(nameof(ClusterData.StartStringData)))
.Append(MlContext.Transforms.Text.ProduceNgrams(nameof(ClusterData.StartStringData)))
.Append(MlContext.Transforms.NormalizeLpNorm(nameof(ClusterData.StartStringData)))
.Append(MlContext.Transforms.Text.NormalizeText(nameof(ClusterData.EndStringData))
.Append(MlContext.Transforms.Text.TokenizeIntoWords(nameof(ClusterData.EndStringData)))
.Append(MlContext.Transforms.Text.RemoveDefaultStopWords(nameof(ClusterData.EndStringData)))
.Append(MlContext.Transforms.Conversion.MapValueToKey(nameof(ClusterData.EndStringData)))
.Append(MlContext.Transforms.Text.ProduceNgrams(nameof(ClusterData.EndStringData)))
.Append(MlContext.Transforms.NormalizeLpNorm(nameof(ClusterData.EndStringData)))
.Append(MlContext.Transforms.Concatenate(featuresColumnName, nameof(ClusterData.StartStringData), nameof(ClusterData.EndStringData))));
var trainer = MlContext.Clustering.Trainers.KMeans(featureColumnName: featuresColumnName, numberOfClusters: 5);
var trainingPipeline = pipeline.Append(trainer);
var trainedModel = trainingPipeline.Fit(trainingDataView);
Logger <TrainerCommandLineOptions> .Debug($"Model trained in {DateTime.Now.Subtract(startDate).TotalSeconds} seconds", options);
var predictions = trainedModel.Transform(testingDataView);
var metrics = MlContext.Clustering.Evaluate(predictions, scoreColumnName: "Score", featureColumnName: featuresColumnName);
Logger <TrainerCommandLineOptions> .Debug($"Average Distance: {metrics.AverageDistance} | Davides Bouldin Index: {metrics.DaviesBouldinIndex}", options);
SaveModel(trainedModel, trainingDataView.Schema, options);
return(true);
}
protected string SaveModel(ITransformer trainedModel, DataViewSchema schema, TrainerCommandLineOptions options)
{
try
{
using (var fileStream = new FileStream(OutputModelPath, FileMode.Create, FileAccess.Write, FileShare.Write))
{
MlContext.Model.Save(trainedModel, schema, fileStream);
}
return(OutputModelPath);
}
catch (Exception ex)
{
Log.Error($"Failure in saving model to {OutputModelPath} due to {ex}");
return(null);
}
}
protected string FeatureExtractFolder(TrainerCommandLineOptions options)
{
var fileName = Path.Combine(AppContext.BaseDirectory, $"{DateTime.Now.Ticks}.txt");
var files = Directory.GetFiles(options.FolderOfData);
Logger <TrainerCommandLineOptions> .Debug($"{files.Length} Files found for training...", options);
var stopWatch = DateTime.Now;
var extractions = new ConcurrentQueue <string>();
var classifications = new ConcurrentQueue <FileGroupType>();
Parallel.ForEach(files, file =>
{
var response = new ClassifierResponseItem(File.ReadAllBytes(file), file, true);
var(data, output) = FeatureExtraction(response);
classifications.Enqueue(response.FileGroup);
extractions.Enqueue(output);
});
File.WriteAllText(fileName, string.Join(System.Environment.NewLine, extractions));
var featureBreakdown = (from classification in classifications.GroupBy(a => a).Select(a => a.Key)
let count = classifications.Count(a => a == classification)
let percentage = Math.Round((double)count / files.Length * 100.0, 0)
select $"{classification}: {(double) count} ({percentage}%)").ToList();
Logger <TrainerCommandLineOptions> .Debug(string.Join("|", featureBreakdown), options);
Logger <TrainerCommandLineOptions> .Debug($"Feature Extraction took {DateTime.Now.Subtract(stopWatch).TotalSeconds} seconds", options);
return(fileName);
}
public abstract bool TrainModel(TrainerCommandLineOptions options);
public async Task <bool> Run(Hosts host, string serverURL)
{
_host = host;
_serverURL = serverURL;
CheckPendingSubmissions();
var workerHandler = new WorkerHandler(_serverURL);
var work = await workerHandler.GetWorkAsync(_host.Name);
if (work == null)
{
System.Threading.Thread.Sleep(Constants.LOOP_INTERVAL_MS);
return(false);
}
work.Started = true;
work.StartTime = DateTime.Now;
var result = await workerHandler.UpdateWorkAsync(work);
if (!result)
{
System.Threading.Thread.Sleep(Constants.LOOP_INTERVAL_MS);
return(false);
}
if (!Directory.Exists(work.TrainingDataPath))
{
work.Completed = true;
work.Debug = $"Path ({work.TrainingDataPath}) does not exist";
work.CompletedTime = DateTime.Now;
result = await workerHandler.UpdateWorkAsync(work);
if (!result)
{
AddToPending(work);
}
return(false);
}
var options = new TrainerCommandLineOptions
{
FolderOfData = work.TrainingDataPath,
LogLevel = LogLevels.DEBUG
};
var(outputFile, metrics) = (string.Empty, string.Empty);
switch (Enum.Parse <ModelType>(work.ModelType, true))
{
case ModelType.CLASSIFICATION:
(outputFile, metrics) = new ClassificationEngine().TrainModel(options);
break;
case ModelType.CLUSTERING:
(outputFile, metrics) = new ClusteringEngine().TrainModel(options);
break;
}
if (File.Exists(outputFile))
{
work.Model = File.ReadAllBytes(outputFile);
}
work.ModelEvaluationMetrics = metrics;
work.Completed = true;
work.CompletedTime = DateTime.Now;
result = await workerHandler.UpdateWorkAsync(work);
if (result)
{
Console.WriteLine($"Successfully trained model and saved to {outputFile}");
}
else
{
AddToPending(work);
}
return(result);
}
public static TrainerCommandLineOptions Parse(string[] args)
{
TrainerCommandLineOptions options = null;
Option oVerbose = new Option(
"--verbose",
"Enable verbose output",
new Argument <bool>(defaultValue: false));
Option oFolder = new Option(
"--folderofdata",
"Folder containing data to be parsed to build the model",
new Argument <string>());
Option oModelType = new Option(
"--modeltype",
"Model Type",
new Argument <ModelType>());
var rootCommand = new RootCommand
{
Description = "File Trainer builds a model"
};
rootCommand.AddOption(oFolder);
rootCommand.AddOption(oVerbose);
rootCommand.AddOption(oModelType);
rootCommand.TreatUnmatchedTokensAsErrors = true;
rootCommand.Argument.AddValidator(symbolResult =>
{
if (symbolResult.Children["--folderofdata"] is null)
{
return("Folder Path is required");
}
else
{
return(null);
}
});
rootCommand.Handler = CommandHandler.Create <string, bool, ModelType>((folderPath, verbose, modelType) =>
{
if (string.IsNullOrEmpty(folderPath))
{
return;
}
options = new TrainerCommandLineOptions
{
FolderOfData = folderPath,
LogLevel = LogLevels.DEBUG,
ModelType = modelType
};
});
rootCommand.InvokeAsync(args).Wait();
return(options);
}
public abstract (string OutputFile, string Metrics) TrainModel(TrainerCommandLineOptions options);
public async Task <bool> Run(Workers worker, Config config)
{
_worker = worker;
_config = config;
var workerHandler = new WorkerHandler(_config.WebServiceURL, _config.RegistrationKey);
var work = await workerHandler.GetWorkAsync(_worker.Name);
if (work == null)
{
Log.Debug($"No work or connection issues to {_config.WebServiceURL}, waiting until next interval");
System.Threading.Thread.Sleep(Constants.LOOP_INTERVAL_MS);
return(false);
}
work.Started = true;
work.StartTime = DateTime.Now;
var result = await workerHandler.UpdateWorkAsync(work);
if (!result)
{
System.Threading.Thread.Sleep(Constants.LOOP_INTERVAL_MS);
return(false);
}
if (!Directory.Exists(work.TrainingDataPath))
{
work.Completed = true;
work.Debug = $"Path ({work.TrainingDataPath}) does not exist";
work.CompletedTime = DateTime.Now;
result = await workerHandler.UpdateWorkAsync(work);
if (!result)
{
AddToPending(work);
}
return(false);
}
var options = new TrainerCommandLineOptions
{
FolderOfData = work.TrainingDataPath,
LogLevel = LogLevels.DEBUG
};
var featureExtractor = Assembly.Load(work.FeatureExtractorBytes);
if (featureExtractor == null)
{
work.Debug = "Feature Extractor Assembly was not piped to the worker";
return(false);
}
var extractor = featureExtractor.GetTypes()
.Where(a => a.BaseType == typeof(BasePrediction) && !a.IsAbstract)
.Select(a => ((BasePrediction)Activator.CreateInstance(a)))
.FirstOrDefault(a => a.MODEL_NAME == work.ModelType);
if (extractor == null)
{
work.Debug = $"Failed to load {work.ModelType} from piped in assembly";
return(false);
}
var(outputFile, metrics) = extractor.TrainModel(options);
if (File.Exists(outputFile))
{
work.Model = File.ReadAllBytes(outputFile);
}
work.ModelEvaluationMetrics = metrics;
work.Completed = true;
work.CompletedTime = DateTime.Now;
result = await workerHandler.UpdateWorkAsync(work);
if (result)
{
Log.Debug($"{work.ID}.{work.Name} - was successfully trained and saved to {outputFile}");
Console.WriteLine($"Successfully trained model and saved to {outputFile}");
}
else
{
AddToPending(work);
}
return(result);
}