/// <summary>
/// Starts model training
/// </summary>
/// <param name="modelId"></param>
/// <param name="parameters"></param>
/// <param name="cancellationToken"></param>
/// <returns></returns>
private async Task <bool> TrainModelAsync(Guid modelId, ModelTrainingParameters parameters,
CancellationToken cancellationToken)
{
// create an auto reset event to ensure that model status updates happen one at a time
var ongoingUpdateEvent = new AutoResetEvent(true);
// create a progress messages event handler for updating the model status message in the registry
Action <string> progressMessageHandler = progressMessage => ModelTrainingProgressMessagesEventHandler(
progressMessage, modelId, ongoingUpdateEvent, cancellationToken);
Trace.TraceInformation($"Starting model '{modelId}' training");
ModelTrainResult result = await _modelsProvider.TrainAsync(
modelId, parameters, progressMessageHandler, cancellationToken);
// get the model status
ModelStatus newModelStatus = result.IsCompletedSuccessfuly ? ModelStatus.Completed : ModelStatus.Failed;
Trace.TraceInformation($"Model training completed with status '{newModelStatus}'");
Trace.TraceInformation("Extracting model statistics from the model training result");
ModelStatistics modelStatistics = CreateModelStatistics(result, parameters);
Trace.TraceInformation("Wait for any ongoing model status message updates before updating the final status");
ongoingUpdateEvent.WaitOne();
Trace.TraceInformation("Update the model status and statistics to the registry");
await _modelsRegistry.UpdateModelAsync(modelId, cancellationToken,
newModelStatus, result.CompletionMessage, modelStatistics);
return(result.IsCompletedSuccessfuly);
}
public async Task RunAsync(CancellationToken token)
{
_logger.LogInformation("RfmTrainingWorker.RunAsync");
IReadOnlyList <string> tableNames = _options.TableNames;
List <Task <TableStatistics> > tableStatisticsTasks = new List <Task <TableStatistics> >(tableNames.Count);
foreach (string tableName in tableNames)
{
tableStatisticsTasks.Add(this._tableStore.GetTableStatisticsAsync(tableName, token));
}
TableStatistics[] tableStatisticsArray = await Task.WhenAll(tableStatisticsTasks).ConfigureAwait(false);
List <TableDefinition> tableDefinitionList = new List <TableDefinition>(tableStatisticsTasks.Count);
for (int index = 0; index < tableStatisticsTasks.Count; ++index)
{
TableStatistics result = tableStatisticsTasks[index].Result;
if (result == null)
{
this._logger.LogWarning(string.Format("Statistics data for {0} table could not be retrieved. It will not participate in model training.", (object)tableNames[index]));
}
else
{
tableDefinitionList.Add(result.Definition);
}
}
ModelStatistics modelStatistics = await _model.TrainAsync(_options.SchemaName, token, tableDefinitionList.ToArray()).ConfigureAwait(false);
if (modelStatistics != null)
{
await UpdateRfmFacets(modelStatistics as RfmStatistics, token);
}
}
/// <summary>
/// Starts model training
/// </summary>
/// <param name="modelId"></param>
/// <param name="parameters"></param>
/// <param name="cancellationToken"></param>
/// <returns></returns>
private async Task <bool> TrainModelAsync(Guid modelId, ModelTrainingParameters parameters,
CancellationToken cancellationToken)
{
// create an auto reset event to ensure that model status updates happen one at a time
var ongoingUpdateEvent = new AutoResetEvent(true);
// create a progress messages event handler for updating the model status message in the registry
Action <string> progressMessageHandler = progressMessage => ModelTrainingProgressMessagesEventHandler(
progressMessage, modelId, ongoingUpdateEvent, cancellationToken);
Trace.TraceInformation($"Iniciando entrenamiento del modelo '{modelId}'.");
ModelTrainResult result = await _modelsProvider.TrainAsync(
modelId, parameters, progressMessageHandler, cancellationToken);
// get the model status
ModelStatus newModelStatus = result.IsCompletedSuccessfuly ? ModelStatus.Completed : ModelStatus.Failed;
Trace.TraceInformation($"Entrenamiento del Modelo completado con estado '{newModelStatus}'");
Trace.TraceInformation("Extrayendo estadísticas de los resultados de entrenamiento del modelo");
ModelStatistics modelStatistics = CreateModelStatistics(result, parameters);
Trace.TraceInformation("Espere a que se actualice el mensaje de estado del modelo en curso antes de actualizar el estado final");
ongoingUpdateEvent.WaitOne();
Trace.TraceInformation("Actualice el estado del modelo y estadisticas al registro");
await _modelsRegistry.UpdateModelAsync(modelId, cancellationToken,
newModelStatus, result.CompletionMessage, modelStatistics);
return(result.IsCompletedSuccessfuly);
}
/// <summary>
/// Create model statistics out of the model training result
/// </summary>
private static ModelStatistics CreateModelStatistics(ModelTrainResult result, ModelTrainingParameters parameters)
{
var statistics = new ModelStatistics
{
// set the total duration
TotalDuration = result.Duration.TotalDuration,
// set the core training duration
TrainingDuration = result.Duration.TrainingDuration,
// set the storing user history duration
StoringUserHistoryDuration = result.Duration.StoringUserHistoryDuration,
// create the catalog parsing report
CatalogParsingReport = CreateParsingReport(result.CatalogFilesParsingReport,
result.Duration.CatalogParsingDuration,
string.IsNullOrWhiteSpace(parameters.CatalogFileRelativePath)
? null
: Path.GetDirectoryName(parameters.CatalogFileRelativePath)),
// create the usage files parsing report
UsageEventsParsingReport = CreateParsingReport(result.UsageFilesParsingReport,
result.Duration.UsageFilesParsingDuration,
parameters.UsageRelativePath),
// set the number of items in catalog
NumberOfCatalogItems = result.CatalogItemsCount,
// set the number of valid items in usage files
NumberOfUsageItems = result.UniqueItemsCount,
// set the number of unique users in usage files
NumberOfUsers = result.UniqueUsersCount,
// set the catalog coverage when applicable
CatalogCoverage =
result.CatalogItemsCount != null &&
result.CatalogItemsCount != 0
? (double)result.UniqueItemsCount / result.CatalogItemsCount
: null,
// set the catalog features weights, if calculated
CatalogFeatureWeights = result.CatalogFeatureWeights?.Count > 0 ? result.CatalogFeatureWeights : null
};
// set the evaluation statistics if available
if (!string.IsNullOrWhiteSpace(parameters.EvaluationUsageRelativePath))
{
// create evaluation result
statistics.EvaluationResult = new ModelEvaluationResult
{
// set the evaluation duration
Duration = result.Duration.EvaluationDuration,
// set the evaluation result
Metrics = result.ModelMetrics,
// create the evaluation usage files parsing report
EvaluationUsageEventsParsingReport =
CreateParsingReport(result.EvaluationFilesParsingReport,
result.Duration.EvaluationUsageFilesParsingDuration,
parameters.EvaluationUsageRelativePath)
};
}
return(statistics);
}
/// <summary>
/// Updates a model by overriding the provided non-null properties.
/// </summary>
/// <param name="modelId">The id of the model to update</param>
/// <param name="cancellationToken">The cancellation token assigned for the operation.</param>
/// <param name="status">A new status to update</param>
/// <param name="statusMessage">A status message</param>
/// <param name="statistics">Optional new model statistics to update</param>
public async Task UpdateModelAsync(Guid modelId, CancellationToken cancellationToken,
ModelStatus?status = null, string statusMessage = null, ModelStatistics statistics = null)
{
ThrowIfDisposed();
try
{
// create a model entity
var modelEntity = new ModelTableEntity(modelId, null, statistics)
{
ETag = "*"
};
// set the model status (if provided)
if (status.HasValue)
{
modelEntity.ModelStatus = status.Value.ToString();
}
// set the model status message (if provided)
if (statusMessage != null)
{
if (statusMessage.Length > MaxAllowedStatusMessageLength)
{
statusMessage = statusMessage.Substring(0, MaxAllowedStatusMessageLength) + "... [trimmed]";
}
modelEntity.StatusMessage = statusMessage;
}
Trace.TraceInformation($"Updating model '{modelId}' properties in the table");
// override only the non-null properties of the update model entity to the table
await _modelsTable.MergeEntityAsync(modelEntity, cancellationToken);
}
catch (StorageException storageException)
{
var exception = new Exception($"Exception while trying to update model entity ({modelId}) to the table",
storageException);
Trace.TraceError(exception.ToString());
throw exception;
}
}
public async Task UpdateModelAsyncTest()
{
ITable table = Substitute.For <ITable>();
var modelsRegistry = new ModelsRegistry(table);
table.MergeEntityAsync(Arg.Any <ModelTableEntity>(), CancellationToken.None).Returns(Task.FromResult(true));
var modelId = Guid.NewGuid();
// Scenario 1: All variables null. nothing should be overridden.
await modelsRegistry.UpdateModelAsync(modelId, CancellationToken.None);
await table.Received(1).MergeEntityAsync(
Arg.Is <ModelTableEntity>(
me => Guid.Parse(me.RowKey) == modelId && me.ModelStatus == null && me.StatusMessage == null && me.ModelStatistics == null),
CancellationToken.None);
// Scenario 2: Only status needs to be updated
table.ClearReceivedCalls();
await modelsRegistry.UpdateModelAsync(modelId, CancellationToken.None, ModelStatus.InProgress);
await table.Received(1).MergeEntityAsync(
Arg.Is <ModelTableEntity>(
me =>
Guid.Parse(me.RowKey) == modelId && me.ModelStatus == ModelStatus.InProgress.ToString() &&
me.StatusMessage == null && me.ModelStatistics == null), CancellationToken.None);
// Scenario 3: Only status message needs to be updated
table.ClearReceivedCalls();
await modelsRegistry.UpdateModelAsync(modelId, CancellationToken.None, null, "test");
await table.Received(1).MergeEntityAsync(
Arg.Is <ModelTableEntity>(
me => Guid.Parse(me.RowKey) == modelId && me.ModelStatus == null && me.StatusMessage == "test" && me.ModelStatistics == null),
CancellationToken.None);
// Scenario 4: Only statistics needs to be updated
var modelStatistics = new ModelStatistics
{
NumberOfCatalogItems = 10
};
table.ClearReceivedCalls();
await modelsRegistry.UpdateModelAsync(modelId, CancellationToken.None, null, null, modelStatistics);
await table.Received(1).MergeEntityAsync(
Arg.Is <ModelTableEntity>(
me =>
Guid.Parse(me.RowKey) == modelId && me.ModelStatus == null && me.StatusMessage == null &&
me.ModelStatistics == JsonConvert.SerializeObject(modelStatistics)),
CancellationToken.None);
// Scenario 5: everything needs to be updated
table.ClearReceivedCalls();
await modelsRegistry.UpdateModelAsync(modelId, CancellationToken.None, ModelStatus.Completed, "test", modelStatistics);
await table.Received(1).MergeEntityAsync(
Arg.Is <ModelTableEntity>(
me =>
Guid.Parse(me.RowKey) == modelId && me.ModelStatus == ModelStatus.Completed.ToString() &&
me.StatusMessage == "test" && me.ModelStatistics == JsonConvert.SerializeObject(modelStatistics)),
CancellationToken.None);
}
/// <summary>
/// Creates a new instance of the <see cref="ModelTableEntity"/> class
/// </summary>
/// <param name="modelId">The model id</param>
/// <param name="modelParameters">The training parameters of the model</param>
/// <param name="modelStatistics">The model training statistics</param>
public ModelTableEntity(Guid modelId, ModelTrainingParameters modelParameters = null, ModelStatistics modelStatistics = null)
: this()
{
RowKey = modelId.ToString();
if (modelParameters != null)
{
ModelParameters = JsonConvert.SerializeObject(modelParameters);
}
if (modelStatistics != null)
{
ModelStatistics = JsonConvert.SerializeObject(modelStatistics);
}
}
