private async void CheckPendingSubmissions()
{
var db = new LiteDBDatabase();
var pendingJobs = db.GetPendingSubmissions();
if (!pendingJobs.Any())
{
Log.Debug("No Pending Jobs found");
return;
}
Log.Debug($"{pendingJobs.Count} pending jobs found...");
var workerHandler = new WorkerHandler(_serverURL);
foreach (var pJob in pendingJobs)
{
Jobs job = null;
try
{
job = JsonConvert.DeserializeObject <Jobs>(pJob.JobJSON);
}
catch (Exception ex)
{
Log.Error($"For Job ID {pJob.ID}, could not parse {pJob.JobJSON} into a Jobs object due to {ex}");
}
if (job == null)
{
Log.Error($"Job was null - removing {pJob.ID} from Queue");
_db.RemoveOfflineSubmission(pJob.ID);
continue;
}
var result = await workerHandler.UpdateWorkAsync(job);
if (result)
{
Log.Debug($"{job.ID} was successfully uploaded");
_db.RemoveOfflineSubmission(pJob.ID);
continue;
}
Log.Debug($"{job.ID} was not able to be uploaded - will retry at a later date and time");
}
}
private async void BwCheckin_DoWork(object sender, DoWorkEventArgs e)
{
var submissionManager = new SubmissionManager(new DatabaseManager(_db, null));
var pendingJobs = submissionManager.GetPendingSubmissions();
if (!pendingJobs.Any())
{
Log.Debug("No Pending Jobs found");
return;
}
Log.Debug($"{pendingJobs.Count} pending jobs found...");
var workerHandler = new WorkerHandler(_config.WebServiceURL, _config.RegistrationKey);
foreach (var pJob in pendingJobs)
{
Jobs job = null;
try
{
job = JsonConvert.DeserializeObject <Jobs>(pJob.JobJSON);
} catch (Exception ex)
{
Log.Error($"For Job ID {pJob.ID}, could not parse {pJob.JobJSON} into a Jobs object due to {ex}");
}
if (job == null)
{
Log.Error($"Job was null - removing {pJob.ID} from Queue");
submissionManager.RemoveOfflineSubmission(pJob.ID);
continue;
}
var result = await workerHandler.UpdateWorkAsync(job);
if (result)
{
Log.Debug($"{job.ID} was successfully uploaded");
submissionManager.RemoveOfflineSubmission(pJob.ID);
continue;
}
Log.Debug($"{job.ID} was not able to be uploaded - will retry at a later date and time");
}
}
public async Task <bool> Run(Hosts host, string serverURL)
{
_host = host;
_serverURL = serverURL;
CheckPendingSubmissions();
var workerHandler = new WorkerHandler(_serverURL);
var work = await workerHandler.GetWorkAsync(_host.Name);
if (work == null)
{
System.Threading.Thread.Sleep(Constants.LOOP_INTERVAL_MS);
return(false);
}
work.Started = true;
work.StartTime = DateTime.Now;
var result = await workerHandler.UpdateWorkAsync(work);
if (!result)
{
System.Threading.Thread.Sleep(Constants.LOOP_INTERVAL_MS);
return(false);
}
if (!Directory.Exists(work.TrainingDataPath))
{
work.Completed = true;
work.Debug = $"Path ({work.TrainingDataPath}) does not exist";
work.CompletedTime = DateTime.Now;
result = await workerHandler.UpdateWorkAsync(work);
if (!result)
{
AddToPending(work);
}
return(false);
}
var options = new TrainerCommandLineOptions
{
FolderOfData = work.TrainingDataPath,
LogLevel = LogLevels.DEBUG
};
var(outputFile, metrics) = (string.Empty, string.Empty);
switch (Enum.Parse <ModelType>(work.ModelType, true))
{
case ModelType.CLASSIFICATION:
(outputFile, metrics) = new ClassificationEngine().TrainModel(options);
break;
case ModelType.CLUSTERING:
(outputFile, metrics) = new ClusteringEngine().TrainModel(options);
break;
}
if (File.Exists(outputFile))
{
work.Model = File.ReadAllBytes(outputFile);
}
work.ModelEvaluationMetrics = metrics;
work.Completed = true;
work.CompletedTime = DateTime.Now;
result = await workerHandler.UpdateWorkAsync(work);
if (result)
{
Console.WriteLine($"Successfully trained model and saved to {outputFile}");
}
else
{
AddToPending(work);
}
return(result);
}
public async Task <bool> Run(Workers worker, Config config)
{
_worker = worker;
_config = config;
var workerHandler = new WorkerHandler(_config.WebServiceURL, _config.RegistrationKey);
var work = await workerHandler.GetWorkAsync(_worker.Name);
if (work == null)
{
Log.Debug($"No work or connection issues to {_config.WebServiceURL}, waiting until next interval");
System.Threading.Thread.Sleep(Constants.LOOP_INTERVAL_MS);
return(false);
}
work.Started = true;
work.StartTime = DateTime.Now;
var result = await workerHandler.UpdateWorkAsync(work);
if (!result)
{
System.Threading.Thread.Sleep(Constants.LOOP_INTERVAL_MS);
return(false);
}
if (!Directory.Exists(work.TrainingDataPath))
{
work.Completed = true;
work.Debug = $"Path ({work.TrainingDataPath}) does not exist";
work.CompletedTime = DateTime.Now;
result = await workerHandler.UpdateWorkAsync(work);
if (!result)
{
AddToPending(work);
}
return(false);
}
var options = new TrainerCommandLineOptions
{
FolderOfData = work.TrainingDataPath,
LogLevel = LogLevels.DEBUG
};
var featureExtractor = Assembly.Load(work.FeatureExtractorBytes);
if (featureExtractor == null)
{
work.Debug = "Feature Extractor Assembly was not piped to the worker";
return(false);
}
var extractor = featureExtractor.GetTypes()
.Where(a => a.BaseType == typeof(BasePrediction) && !a.IsAbstract)
.Select(a => ((BasePrediction)Activator.CreateInstance(a)))
.FirstOrDefault(a => a.MODEL_NAME == work.ModelType);
if (extractor == null)
{
work.Debug = $"Failed to load {work.ModelType} from piped in assembly";
return(false);
}
var(outputFile, metrics) = extractor.TrainModel(options);
if (File.Exists(outputFile))
{
work.Model = File.ReadAllBytes(outputFile);
}
work.ModelEvaluationMetrics = metrics;
work.Completed = true;
work.CompletedTime = DateTime.Now;
result = await workerHandler.UpdateWorkAsync(work);
if (result)
{
Log.Debug($"{work.ID}.{work.Name} - was successfully trained and saved to {outputFile}");
Console.WriteLine($"Successfully trained model and saved to {outputFile}");
}
else
{
AddToPending(work);
}
return(result);
}
