public void WithColumnCombined()
{
var t1 = ModelsRegistry.GetTable <MyModel>();
var t2 = ModelsRegistry.GetTable <MyModel2>();
var rq1 = new RawQuery();
rq1.AppendColumn(NodeVisitor.EvaluateFuncExpressionToColumn <MyModel>(x => x.Name), new RawQuery.TableNamePlaceholder(t1, "M1"));
rq1.Append(" = 123");
rq1.ApplyAlias("M1", "t1");
var rq2 = new RawQuery();
rq2.AppendColumn(NodeVisitor.EvaluateFuncExpressionToColumn <MyModel2>(x => x.Name1), new RawQuery.TableNamePlaceholder(t2, "M2"));
rq2.Append(" = 456");
rq2.ApplyAlias("M2", "t2");
var rq = new RawQuery();
rq.Append(rq1);
rq.Append(", ");
rq.Append(rq2);
Utils.AssertRawQueryWithAliases(rq, @"t1.""name"" = 123, t2.name1 = 456");
}
public async Task <IHttpActionResult> GetModels(CancellationToken cancellationToken)
{
Trace.TraceVerbose("Listing all models in the registry");
ModelsRegistry modelsRegistry = WebAppContext.ModelsRegistry;
IList <Model> models = await modelsRegistry.ListModelsAsync(cancellationToken);
return(Ok(models));
}
public static SelectFromBuilder FromCombinedExpression <TCombinedModel, TNewModel>(TablesList tablesList, Expression <Func <TCombinedModel, TNewModel> > prBody)
{
var builder = new SelectFromBuilder();
builder.LeftJoinsExpressions = tablesList.JoinExpressions;
var options = new EvaluationOptions();
foreach (var tablePlaceholder in tablesList.Tables)
{
builder.AddTable(tablePlaceholder);
// options.ParameterToTableAlias.Add(tablePlaceholder.Name, tablePlaceholder);
}
var tableToPlaceholder = tablesList.Tables.ToDictionary(x => x.Name);
TypedExpression exp;
switch (prBody.Body)
{
/* For:
* .Select(x => new {
* M1 = x.M1,
* M2_name = x.M2.Name1,
* M3_calc = x.M2.Id * 2,
* })
*/
case NewExpression newExpression:
{
IEnumerable <Expression> args = newExpression.Arguments;
var resultProcessor = new AnonymousTypeSelectResultProcessor <TNewModel>();
builder.SelectResultProcessor = resultProcessor;
foreach (var argExpression in args)
{
// Member is Table (like M1 = x.M1)
if (argExpression is MemberExpression memberExpression && ModelsRegistry.IsTable(memberExpression.Type))
{
var tablePlaceholder = tableToPlaceholder[memberExpression.Member.Name];
var table = tablePlaceholder.Table;
var tableTestRawQuery = new RawQuery();
tableTestRawQuery.AppendTable(tablePlaceholder);
tableTestRawQuery.Append(" IS NULL");
builder.AddSelectPart(tableTestRawQuery, KDPgValueTypeInstances.Boolean);
foreach (var column in table.Columns)
{
var rq = new RawQuery();
rq.AppendColumn(column, tablePlaceholder);
builder.AddSelectPart(rq, column.Type);
}
resultProcessor.AddModelEntry(table);
}
public async Task <IHttpActionResult> TrainNewModel(CancellationToken cancellationToken, [FromBody] ModelParameters modelParameters)
{
// validate input
if (modelParameters == null)
{
var message = $"Invalid format. Expected a valid '{nameof(ModelParameters)}' JSON";
Trace.TraceVerbose(message);
return(BadRequest(message));
}
if (!ModelState.IsValid)
{
return(BadRequest(ModelState));
}
ModelsRegistry modelsRegistry = WebAppContext.ModelsRegistry;
Trace.TraceVerbose("Converting the model parameters to trainer settings, using default values where needed");
var @default = ModelTrainingParameters.Default;
var settings = new ModelTrainingParameters
{
BlobContainerName = modelParameters.BlobContainerName,
CatalogFileRelativeLocation = modelParameters.CatalogFileRelativeLocation?.Replace('\\', '/'),
UsageFolderRelativeLocation = modelParameters.UsageFolderRelativeLocation?.Replace('\\', '/'),
EvaluationUsageFolderRelativeLocation = modelParameters.EvaluationUsageFolderRelativeLocation?.Replace('\\', '/'),
SupportThreshold = modelParameters.SupportThreshold ?? @default.SupportThreshold,
CooccurrenceUnit = modelParameters.CooccurrenceUnit ?? @default.CooccurrenceUnit,
SimilarityFunction = modelParameters.SimilarityFunction ?? @default.SimilarityFunction,
EnableColdItemPlacement = modelParameters.EnableColdItemPlacement ?? @default.EnableColdItemPlacement,
EnableColdToColdRecommendations =
modelParameters.EnableColdToColdRecommendations ?? @default.EnableColdToColdRecommendations,
EnableUserAffinity = modelParameters.EnableUserAffinity ?? @default.EnableUserAffinity,
EnableUserToItemRecommendations =
modelParameters.EnableUserToItemRecommendations ?? @default.EnableUserToItemRecommendations,
AllowSeedItemsInRecommendations =
modelParameters.AllowSeedItemsInRecommendations ?? @default.AllowSeedItemsInRecommendations,
EnableBackfilling = modelParameters.EnableBackfilling ?? @default.EnableBackfilling,
DecayPeriodInDays = modelParameters.DecayPeriodInDays ?? @default.DecayPeriodInDays
};
Trace.TraceInformation("Creating new model in registry");
Model model = await modelsRegistry.CreateModelAsync(settings, modelParameters.Description, cancellationToken);
Trace.TraceInformation($"Queueing a new train model message to the queue for model id {model.Id}");
ModelQueueMessage modelQueueMessage = new ModelQueueMessage {
ModelId = model.Id
};
await WebAppContext.TrainModelQueue.AddMessageAsync(modelQueueMessage, cancellationToken);
// return the URL to the created model
return(CreatedAtRoute(nameof(GetModel), new { modelId = model.Id }, model));
}
public async Task SetDefaultModelIdAsyncTest()
{
ITable table = Substitute.For <ITable>();
var modelsRegistry = new ModelsRegistry(table);
// Scenario 1: Invalid Model
// Validate set api is not called
table.GetEntityAsync <ModelTableEntity>(Arg.Any <string>(), CancellationToken.None, Arg.Any <string[]>())
.Returns(Task.FromResult <ModelTableEntity>(null));
table.InsertOrReplaceEntityAsync(Arg.Any <ModelTableEntity>(), CancellationToken.None)
.Returns(Task.FromResult(true));
var unknownId = Guid.NewGuid();
bool result = await modelsRegistry.SetDefaultModelIdAsync(unknownId, CancellationToken.None);
Assert.IsFalse(result);
await table.DidNotReceive().InsertOrReplaceEntityAsync(Arg.Any <ModelIdTableEntity>(), Arg.Any <CancellationToken>());
// Scenario 2: Valid Model but Status not Complete
// Validate set api is not called
var knownId = Guid.NewGuid();
table.GetEntityAsync <ModelTableEntity>(knownId.ToString(), CancellationToken.None, Arg.Any <string[]>())
.Returns(
Task.FromResult(new ModelTableEntity(knownId)
{
ModelStatus = ModelStatus.InProgress.ToString()
}));
result = await modelsRegistry.SetDefaultModelIdAsync(knownId, CancellationToken.None);
Assert.IsFalse(result);
await table.DidNotReceive()
.InsertOrReplaceEntityAsync(Arg.Any <ModelIdTableEntity>(), Arg.Any <CancellationToken>());
// Scenario 3: Valid Model with Status as Complete
// Validate set api is called
table.GetEntityAsync <ModelTableEntity>(knownId.ToString(), CancellationToken.None, Arg.Any <string[]>())
.Returns(
Task.FromResult(new ModelTableEntity(knownId)
{
ModelStatus = ModelStatus.Completed.ToString()
}));
table.InsertOrReplaceEntityAsync(Arg.Any <ModelIdTableEntity>(), CancellationToken.None).Returns(Task.FromResult(true));
result = await modelsRegistry.SetDefaultModelIdAsync(knownId, CancellationToken.None);
Assert.IsTrue(result);
await table.Received(1).InsertOrReplaceEntityAsync(
Arg.Is <ModelIdTableEntity>(
entity => entity.RowKey == ModelsRegistry.DefaultModelIdKeyName && entity.ModelId == knownId),
CancellationToken.None);
}
public async Task CreateModelAsyncTest()
{
ITable table = Substitute.For <ITable>();
var modelsRegistry = new ModelsRegistry(table);
table.InsertOrReplaceEntityAsync(Arg.Any <ModelIdTableEntity>(), CancellationToken.None).Returns(Task.FromResult(true));
table.InsertEntityAsync(Arg.Any <ModelTableEntity>(), CancellationToken.None).Returns(Task.FromResult(true));
Model model = await modelsRegistry.CreateModelAsync(ModelTrainingParameters.Default, null, CancellationToken.None);
await table.Received(1).InsertEntityAsync(
Arg.Is <ModelTableEntity>(me => Guid.Parse(me.RowKey) == model.Id && me.ModelStatus == ModelStatus.Created.ToString()),
CancellationToken.None);
}
/// <summary>
/// Creates anew instance of the <see cref="WebJobLogic"/> class.
/// </summary>
/// <param name="modelsProvider">A models provider</param>
/// <param name="modelsRegistry">A model registry</param>
public WebJobLogic(ModelsProvider modelsProvider, ModelsRegistry modelsRegistry)
{
if (modelsProvider == null)
{
throw new ArgumentNullException(nameof(modelsProvider));
}
if (modelsRegistry == null)
{
throw new ArgumentNullException(nameof(modelsRegistry));
}
_modelsProvider = modelsProvider;
_modelsRegistry = modelsRegistry;
}
public async Task <IHttpActionResult> GetDefaultModel(CancellationToken cancellationToken)
{
// set the model id to context
ContextManager.ModelId = "default";
Trace.TraceVerbose("Trying to read the default model from the registry");
ModelsRegistry modelsRegistry = WebAppContext.ModelsRegistry;
Model defaultModel = await modelsRegistry.GetDefaultModelAsync(cancellationToken);
if (defaultModel == null)
{
Trace.TraceInformation("A default model is not defined");
return(NotFound());
}
return(Ok(defaultModel));
}
public async Task ClearDefaultModelIdAsyncTest()
{
ITable table = Substitute.For <ITable>();
var modelsRegistry = new ModelsRegistry(table);
// Scenario 1: Model is set as default. Should return true after clearing.
table.DeleteEntityAsync <ModelIdTableEntity>(ModelsRegistry.DefaultModelIdKeyName, CancellationToken.None)
.Returns(Task.FromResult(true));
Assert.IsTrue(await modelsRegistry.ClearDefaultModelIdAsync(CancellationToken.None));
await table.Received(1).DeleteEntityAsync <ModelIdTableEntity>(ModelsRegistry.DefaultModelIdKeyName, CancellationToken.None);
// Scenario 2: Table is unable to delete entity
table.ClearReceivedCalls();
table.DeleteEntityAsync <ModelIdTableEntity>(Arg.Any <string>(), CancellationToken.None)
.Returns(Task.FromResult(false));
Assert.IsFalse(await modelsRegistry.ClearDefaultModelIdAsync(CancellationToken.None));
await table.Received(1).DeleteEntityAsync <ModelIdTableEntity>(ModelsRegistry.DefaultModelIdKeyName, CancellationToken.None);
}
public async Task GetModelAsyncTest()
{
ITable table = Substitute.For <ITable>();
var modelsRegistry = new ModelsRegistry(table);
var knownModelId = Guid.NewGuid();
var unknownModelId = Guid.NewGuid();
table.GetEntityAsync <ModelTableEntity>(knownModelId.ToString(), CancellationToken.None, Arg.Any <string[]>())
.Returns(Task.FromResult(new ModelTableEntity(knownModelId)));
// Valid Model Id
var model = await modelsRegistry.GetModelAsync(knownModelId, CancellationToken.None);
Assert.IsNotNull(model);
Assert.AreEqual(knownModelId, model.Id);
// Invalid Model Id
model = await modelsRegistry.GetModelAsync(unknownModelId, CancellationToken.None);
Assert.IsNull(model);
}
public async Task GetDefaultModelIdAsyncTest()
{
ITable table = Substitute.For <ITable>();
var modelsRegistry = new ModelsRegistry(table);
// Default model not present in Models
var knownModelId = Guid.NewGuid();
table.GetEntityAsync <ModelIdTableEntity>(ModelsRegistry.DefaultModelIdKeyName, CancellationToken.None,
nameof(ModelIdTableEntity.ModelId))
.Returns(Task.FromResult(new ModelIdTableEntity(ModelsRegistry.DefaultModelIdKeyName, knownModelId)));
var model = await modelsRegistry.GetDefaultModelAsync(CancellationToken.None);
Assert.IsNull(model);
// Default model present in Models
table.GetEntityAsync <ModelTableEntity>(knownModelId.ToString(), CancellationToken.None, Arg.Any <string[]>())
.Returns(Task.FromResult(new ModelTableEntity(knownModelId)));
model = await modelsRegistry.GetDefaultModelAsync(CancellationToken.None);
Assert.IsNotNull(model);
Assert.AreEqual(knownModelId, model.Id);
}
public async Task ListModelsAsyncTest()
{
ITable table = Substitute.For <ITable>();
var modelsRegistry = new ModelsRegistry(table);
table.ListEntitiesAsync <ModelTableEntity>(CancellationToken.None, Arg.Any <string[]>())
.Returns(Task.FromResult <IList <ModelTableEntity> >(new List <ModelTableEntity>()));
var models = await modelsRegistry.ListModelsAsync(CancellationToken.None);
Assert.IsNotNull(models);
Assert.AreEqual(0, models.Count);
table.ListEntitiesAsync <ModelTableEntity>(CancellationToken.None, Arg.Any <string[]>())
.Returns(Task.FromResult <IList <ModelTableEntity> >(new[]
{
new ModelTableEntity(Guid.NewGuid()),
new ModelTableEntity(Guid.NewGuid())
}));
models = await modelsRegistry.ListModelsAsync(CancellationToken.None);
Assert.IsNotNull(models);
Assert.AreEqual(2, models.Count);
}
public async Task <IHttpActionResult> GetModel(CancellationToken cancellationToken, [FromUri] Guid?modelId)
{
if (!modelId.HasValue)
{
var message = $"{nameof(modelId)} is not valid";
Trace.TraceVerbose(message);
return(BadRequest(message));
}
// set the model id to context
ContextManager.ModelId = modelId;
Trace.TraceVerbose($"Trying to read the model '{modelId}' from the registry");
ModelsRegistry modelsRegistry = WebAppContext.ModelsRegistry;
Model model = await modelsRegistry.GetModelAsync(modelId.Value, cancellationToken);
if (model == null)
{
Trace.TraceInformation($"Model with id '{modelId}' does not exists.");
return(NotFound());
}
return(Ok(model));
}
public async Task DeleteModelIfExistsAsyncTest()
{
ITable table = Substitute.For <ITable>();
var modelsRegistry = new ModelsRegistry(table);
var knownModelId = Guid.NewGuid();
table.DeleteEntityAsync <ModelTableEntity>(knownModelId.ToString(), CancellationToken.None)
.Returns(Task.FromResult(true));
bool result = await modelsRegistry.DeleteModelIfExistsAsync(knownModelId, CancellationToken.None);
Assert.IsTrue(result);
await table.Received(1)
.DeleteEntityAsync <ModelTableEntity>(Arg.Is <string>(id => Guid.Parse(id) == knownModelId), CancellationToken.None);
var unknownModelId = Guid.NewGuid();
result = await modelsRegistry.DeleteModelIfExistsAsync(unknownModelId, CancellationToken.None);
Assert.IsFalse(result);
await table.Received(1).DeleteEntityAsync <ModelTableEntity>(
Arg.Is <string>(id => Guid.Parse(id) == unknownModelId), CancellationToken.None);
}
public async Task <IHttpActionResult> SetDefaultModel(CancellationToken cancellationToken, Guid?modelId)
{
if (!modelId.HasValue)
{
var message = $"{nameof(modelId)} is not valid";
Trace.TraceVerbose(message);
return(BadRequest(message));
}
// set the model id to context
ContextManager.ModelId = modelId;
Trace.TraceVerbose($"Trying to set '{modelId}' as the default model in the registry");
ModelsRegistry modelsRegistry = WebAppContext.ModelsRegistry;
bool result = await modelsRegistry.SetDefaultModelIdAsync(modelId.Value, cancellationToken);
if (!result)
{
Trace.TraceInformation($"Failed setting model with id '{modelId}' as the default model");
return(NotFound());
}
return(Ok());
}
public async Task UpdateModelAsyncTest()
{
ITable table = Substitute.For <ITable>();
var modelsRegistry = new ModelsRegistry(table);
table.MergeEntityAsync(Arg.Any <ModelTableEntity>(), CancellationToken.None).Returns(Task.FromResult(true));
var modelId = Guid.NewGuid();
// Scenario 1: All variables null. nothing should be overridden.
await modelsRegistry.UpdateModelAsync(modelId, CancellationToken.None);
await table.Received(1).MergeEntityAsync(
Arg.Is <ModelTableEntity>(
me => Guid.Parse(me.RowKey) == modelId && me.ModelStatus == null && me.StatusMessage == null && me.ModelStatistics == null),
CancellationToken.None);
// Scenario 2: Only status needs to be updated
table.ClearReceivedCalls();
await modelsRegistry.UpdateModelAsync(modelId, CancellationToken.None, ModelStatus.InProgress);
await table.Received(1).MergeEntityAsync(
Arg.Is <ModelTableEntity>(
me =>
Guid.Parse(me.RowKey) == modelId && me.ModelStatus == ModelStatus.InProgress.ToString() &&
me.StatusMessage == null && me.ModelStatistics == null), CancellationToken.None);
// Scenario 3: Only status message needs to be updated
table.ClearReceivedCalls();
await modelsRegistry.UpdateModelAsync(modelId, CancellationToken.None, null, "test");
await table.Received(1).MergeEntityAsync(
Arg.Is <ModelTableEntity>(
me => Guid.Parse(me.RowKey) == modelId && me.ModelStatus == null && me.StatusMessage == "test" && me.ModelStatistics == null),
CancellationToken.None);
// Scenario 4: Only statistics needs to be updated
var modelStatistics = new ModelStatistics
{
NumberOfCatalogItems = 10
};
table.ClearReceivedCalls();
await modelsRegistry.UpdateModelAsync(modelId, CancellationToken.None, null, null, modelStatistics);
await table.Received(1).MergeEntityAsync(
Arg.Is <ModelTableEntity>(
me =>
Guid.Parse(me.RowKey) == modelId && me.ModelStatus == null && me.StatusMessage == null &&
me.ModelStatistics == JsonConvert.SerializeObject(modelStatistics)),
CancellationToken.None);
// Scenario 5: everything needs to be updated
table.ClearReceivedCalls();
await modelsRegistry.UpdateModelAsync(modelId, CancellationToken.None, ModelStatus.Completed, "test", modelStatistics);
await table.Received(1).MergeEntityAsync(
Arg.Is <ModelTableEntity>(
me =>
Guid.Parse(me.RowKey) == modelId && me.ModelStatus == ModelStatus.Completed.ToString() &&
me.StatusMessage == "test" && me.ModelStatistics == JsonConvert.SerializeObject(modelStatistics)),
CancellationToken.None);
}
// ReSharper disable ParameterOnlyUsedForPreconditionCheck.Local
private static void AssertPath <TModel>(PathInfo pi, string columnName, string query, KDPgValueType type)
{
Assert.Equal(type, pi.Expression.Type);
Assert.Equal(query, pi.Expression.RawQuery.ToString());
Assert.Equal(ModelsRegistry.GetTable <TModel>().Columns.Find(x => x.Name == columnName), pi.Column);
}
public RawQuery GetRawQuery()
{
if (IsEmpty)
{
return(RawQuery.Create("SELECT 0"));
}
RawQuery rq = new RawQuery();
var columns = _columns.Count == 0 ? AllColumnsWithoutAutoIncrement : _columns;
rq.Append("INSERT INTO ");
rq.AppendTableName(Table.Name, Table.Schema);
rq.Append("(");
rq.AppendColumnNames(columns.Select(x => x.columnDescriptor.Name));
if (_idColumn != null)
{
if (columns.Count > 0)
{
rq.Append(",");
}
rq.AppendColumnName(_idColumn.Name);
}
rq.Append(")");
rq.Append(" VALUES ");
var first = true;
foreach (var obj in _objects)
{
if (!first)
{
rq.Append(",");
}
rq.Append("(");
for (int i = 0; i < columns.Count; i++)
{
var column = columns[i];
if (i > 0)
{
rq.Append(",");
}
if (column.subquery == null)
{
object val = ModelsRegistry.GetModelValueByColumn(obj, column.columnDescriptor);
var npgValue = PgTypesConverter.ConvertToPgValue(column.columnDescriptor.Type, val);
rq.Append(npgValue);
}
else
{
rq.AppendSurround(column.subquery.GetRawQuery());
}
}
if (_idColumn != null)
{
if (columns.Count > 0)
{
rq.Append(",");
}
rq.Append(ExpressionBuilders.CurrSeqValueOfTable(_idRefColumn).RawQuery);
}
rq.Append(")");
first = false;
}
if (_onInsertConflict == OnInsertConflict.DoNothing)
{
rq.Append(" ON CONFLICT DO NOTHING ");
}
if (_onInsertConflict == OnInsertConflict.DoUpdate)
{
rq.Append(" ON CONFLICT (");
var fields = new FieldListBuilder <TModel>();
_onInsertConflictUpdateFields(fields);
first = true;
foreach (var column in fields.Fields)
{
if (!first)
{
rq.Append(", ");
}
rq.AppendColumnName(column.Name);
first = false;
}
rq.Append(") DO UPDATE SET ");
var updateStatementsBuilder = new UpdateStatementsBuilder <TModel>();
_onInsertConflictUpdate(updateStatementsBuilder);
first = true;
foreach (var(column, typedExpression) in updateStatementsBuilder.UpdateParts)
{
if (!first)
{
rq.Append(", ");
}
rq.AppendColumnName(column.Name)
.Append(" = ")
.Append(typedExpression.RawQuery);
first = false;
}
}
if (TableModel.PrimaryKey != null)
{
rq.Append(" RETURNING ");
rq.AppendColumnName(TableModel.PrimaryKey.Name);
}
rq.Append(";");
if (_outputVariable != null)
{
rq.Append(" SELECT ");
rq.Append(ExpressionBuilders.SetConfigText(_outputVariable, ExpressionBuilders.LastVal(), true).RawQuery);
}
rq.SkipExplicitColumnTableNames();
return(rq);
}
