public void WithColumnCombined() { var t1 = ModelsRegistry.GetTable <MyModel>(); var t2 = ModelsRegistry.GetTable <MyModel2>(); var rq1 = new RawQuery(); rq1.AppendColumn(NodeVisitor.EvaluateFuncExpressionToColumn <MyModel>(x => x.Name), new RawQuery.TableNamePlaceholder(t1, "M1")); rq1.Append(" = 123"); rq1.ApplyAlias("M1", "t1"); var rq2 = new RawQuery(); rq2.AppendColumn(NodeVisitor.EvaluateFuncExpressionToColumn <MyModel2>(x => x.Name1), new RawQuery.TableNamePlaceholder(t2, "M2")); rq2.Append(" = 456"); rq2.ApplyAlias("M2", "t2"); var rq = new RawQuery(); rq.Append(rq1); rq.Append(", "); rq.Append(rq2); Utils.AssertRawQueryWithAliases(rq, @"t1.""name"" = 123, t2.name1 = 456"); }
public async Task <IHttpActionResult> GetModels(CancellationToken cancellationToken) { Trace.TraceVerbose("Listing all models in the registry"); ModelsRegistry modelsRegistry = WebAppContext.ModelsRegistry; IList <Model> models = await modelsRegistry.ListModelsAsync(cancellationToken); return(Ok(models)); }
public static SelectFromBuilder FromCombinedExpression <TCombinedModel, TNewModel>(TablesList tablesList, Expression <Func <TCombinedModel, TNewModel> > prBody) { var builder = new SelectFromBuilder(); builder.LeftJoinsExpressions = tablesList.JoinExpressions; var options = new EvaluationOptions(); foreach (var tablePlaceholder in tablesList.Tables) { builder.AddTable(tablePlaceholder); // options.ParameterToTableAlias.Add(tablePlaceholder.Name, tablePlaceholder); } var tableToPlaceholder = tablesList.Tables.ToDictionary(x => x.Name); TypedExpression exp; switch (prBody.Body) { /* For: * .Select(x => new { * M1 = x.M1, * M2_name = x.M2.Name1, * M3_calc = x.M2.Id * 2, * }) */ case NewExpression newExpression: { IEnumerable <Expression> args = newExpression.Arguments; var resultProcessor = new AnonymousTypeSelectResultProcessor <TNewModel>(); builder.SelectResultProcessor = resultProcessor; foreach (var argExpression in args) { // Member is Table (like M1 = x.M1) if (argExpression is MemberExpression memberExpression && ModelsRegistry.IsTable(memberExpression.Type)) { var tablePlaceholder = tableToPlaceholder[memberExpression.Member.Name]; var table = tablePlaceholder.Table; var tableTestRawQuery = new RawQuery(); tableTestRawQuery.AppendTable(tablePlaceholder); tableTestRawQuery.Append(" IS NULL"); builder.AddSelectPart(tableTestRawQuery, KDPgValueTypeInstances.Boolean); foreach (var column in table.Columns) { var rq = new RawQuery(); rq.AppendColumn(column, tablePlaceholder); builder.AddSelectPart(rq, column.Type); } resultProcessor.AddModelEntry(table); }
public async Task <IHttpActionResult> TrainNewModel(CancellationToken cancellationToken, [FromBody] ModelParameters modelParameters) { // validate input if (modelParameters == null) { var message = $"Invalid format. Expected a valid '{nameof(ModelParameters)}' JSON"; Trace.TraceVerbose(message); return(BadRequest(message)); } if (!ModelState.IsValid) { return(BadRequest(ModelState)); } ModelsRegistry modelsRegistry = WebAppContext.ModelsRegistry; Trace.TraceVerbose("Converting the model parameters to trainer settings, using default values where needed"); var @default = ModelTrainingParameters.Default; var settings = new ModelTrainingParameters { BlobContainerName = modelParameters.BlobContainerName, CatalogFileRelativeLocation = modelParameters.CatalogFileRelativeLocation?.Replace('\\', '/'), UsageFolderRelativeLocation = modelParameters.UsageFolderRelativeLocation?.Replace('\\', '/'), EvaluationUsageFolderRelativeLocation = modelParameters.EvaluationUsageFolderRelativeLocation?.Replace('\\', '/'), SupportThreshold = modelParameters.SupportThreshold ?? @default.SupportThreshold, CooccurrenceUnit = modelParameters.CooccurrenceUnit ?? @default.CooccurrenceUnit, SimilarityFunction = modelParameters.SimilarityFunction ?? @default.SimilarityFunction, EnableColdItemPlacement = modelParameters.EnableColdItemPlacement ?? @default.EnableColdItemPlacement, EnableColdToColdRecommendations = modelParameters.EnableColdToColdRecommendations ?? @default.EnableColdToColdRecommendations, EnableUserAffinity = modelParameters.EnableUserAffinity ?? @default.EnableUserAffinity, EnableUserToItemRecommendations = modelParameters.EnableUserToItemRecommendations ?? @default.EnableUserToItemRecommendations, AllowSeedItemsInRecommendations = modelParameters.AllowSeedItemsInRecommendations ?? @default.AllowSeedItemsInRecommendations, EnableBackfilling = modelParameters.EnableBackfilling ?? @default.EnableBackfilling, DecayPeriodInDays = modelParameters.DecayPeriodInDays ?? @default.DecayPeriodInDays }; Trace.TraceInformation("Creating new model in registry"); Model model = await modelsRegistry.CreateModelAsync(settings, modelParameters.Description, cancellationToken); Trace.TraceInformation($"Queueing a new train model message to the queue for model id {model.Id}"); ModelQueueMessage modelQueueMessage = new ModelQueueMessage { ModelId = model.Id }; await WebAppContext.TrainModelQueue.AddMessageAsync(modelQueueMessage, cancellationToken); // return the URL to the created model return(CreatedAtRoute(nameof(GetModel), new { modelId = model.Id }, model)); }
public async Task SetDefaultModelIdAsyncTest() { ITable table = Substitute.For <ITable>(); var modelsRegistry = new ModelsRegistry(table); // Scenario 1: Invalid Model // Validate set api is not called table.GetEntityAsync <ModelTableEntity>(Arg.Any <string>(), CancellationToken.None, Arg.Any <string[]>()) .Returns(Task.FromResult <ModelTableEntity>(null)); table.InsertOrReplaceEntityAsync(Arg.Any <ModelTableEntity>(), CancellationToken.None) .Returns(Task.FromResult(true)); var unknownId = Guid.NewGuid(); bool result = await modelsRegistry.SetDefaultModelIdAsync(unknownId, CancellationToken.None); Assert.IsFalse(result); await table.DidNotReceive().InsertOrReplaceEntityAsync(Arg.Any <ModelIdTableEntity>(), Arg.Any <CancellationToken>()); // Scenario 2: Valid Model but Status not Complete // Validate set api is not called var knownId = Guid.NewGuid(); table.GetEntityAsync <ModelTableEntity>(knownId.ToString(), CancellationToken.None, Arg.Any <string[]>()) .Returns( Task.FromResult(new ModelTableEntity(knownId) { ModelStatus = ModelStatus.InProgress.ToString() })); result = await modelsRegistry.SetDefaultModelIdAsync(knownId, CancellationToken.None); Assert.IsFalse(result); await table.DidNotReceive() .InsertOrReplaceEntityAsync(Arg.Any <ModelIdTableEntity>(), Arg.Any <CancellationToken>()); // Scenario 3: Valid Model with Status as Complete // Validate set api is called table.GetEntityAsync <ModelTableEntity>(knownId.ToString(), CancellationToken.None, Arg.Any <string[]>()) .Returns( Task.FromResult(new ModelTableEntity(knownId) { ModelStatus = ModelStatus.Completed.ToString() })); table.InsertOrReplaceEntityAsync(Arg.Any <ModelIdTableEntity>(), CancellationToken.None).Returns(Task.FromResult(true)); result = await modelsRegistry.SetDefaultModelIdAsync(knownId, CancellationToken.None); Assert.IsTrue(result); await table.Received(1).InsertOrReplaceEntityAsync( Arg.Is <ModelIdTableEntity>( entity => entity.RowKey == ModelsRegistry.DefaultModelIdKeyName && entity.ModelId == knownId), CancellationToken.None); }
public async Task CreateModelAsyncTest() { ITable table = Substitute.For <ITable>(); var modelsRegistry = new ModelsRegistry(table); table.InsertOrReplaceEntityAsync(Arg.Any <ModelIdTableEntity>(), CancellationToken.None).Returns(Task.FromResult(true)); table.InsertEntityAsync(Arg.Any <ModelTableEntity>(), CancellationToken.None).Returns(Task.FromResult(true)); Model model = await modelsRegistry.CreateModelAsync(ModelTrainingParameters.Default, null, CancellationToken.None); await table.Received(1).InsertEntityAsync( Arg.Is <ModelTableEntity>(me => Guid.Parse(me.RowKey) == model.Id && me.ModelStatus == ModelStatus.Created.ToString()), CancellationToken.None); }
/// <summary> /// Creates anew instance of the <see cref="WebJobLogic"/> class. /// </summary> /// <param name="modelsProvider">A models provider</param> /// <param name="modelsRegistry">A model registry</param> public WebJobLogic(ModelsProvider modelsProvider, ModelsRegistry modelsRegistry) { if (modelsProvider == null) { throw new ArgumentNullException(nameof(modelsProvider)); } if (modelsRegistry == null) { throw new ArgumentNullException(nameof(modelsRegistry)); } _modelsProvider = modelsProvider; _modelsRegistry = modelsRegistry; }
public async Task <IHttpActionResult> GetDefaultModel(CancellationToken cancellationToken) { // set the model id to context ContextManager.ModelId = "default"; Trace.TraceVerbose("Trying to read the default model from the registry"); ModelsRegistry modelsRegistry = WebAppContext.ModelsRegistry; Model defaultModel = await modelsRegistry.GetDefaultModelAsync(cancellationToken); if (defaultModel == null) { Trace.TraceInformation("A default model is not defined"); return(NotFound()); } return(Ok(defaultModel)); }
public async Task ClearDefaultModelIdAsyncTest() { ITable table = Substitute.For <ITable>(); var modelsRegistry = new ModelsRegistry(table); // Scenario 1: Model is set as default. Should return true after clearing. table.DeleteEntityAsync <ModelIdTableEntity>(ModelsRegistry.DefaultModelIdKeyName, CancellationToken.None) .Returns(Task.FromResult(true)); Assert.IsTrue(await modelsRegistry.ClearDefaultModelIdAsync(CancellationToken.None)); await table.Received(1).DeleteEntityAsync <ModelIdTableEntity>(ModelsRegistry.DefaultModelIdKeyName, CancellationToken.None); // Scenario 2: Table is unable to delete entity table.ClearReceivedCalls(); table.DeleteEntityAsync <ModelIdTableEntity>(Arg.Any <string>(), CancellationToken.None) .Returns(Task.FromResult(false)); Assert.IsFalse(await modelsRegistry.ClearDefaultModelIdAsync(CancellationToken.None)); await table.Received(1).DeleteEntityAsync <ModelIdTableEntity>(ModelsRegistry.DefaultModelIdKeyName, CancellationToken.None); }
public async Task GetModelAsyncTest() { ITable table = Substitute.For <ITable>(); var modelsRegistry = new ModelsRegistry(table); var knownModelId = Guid.NewGuid(); var unknownModelId = Guid.NewGuid(); table.GetEntityAsync <ModelTableEntity>(knownModelId.ToString(), CancellationToken.None, Arg.Any <string[]>()) .Returns(Task.FromResult(new ModelTableEntity(knownModelId))); // Valid Model Id var model = await modelsRegistry.GetModelAsync(knownModelId, CancellationToken.None); Assert.IsNotNull(model); Assert.AreEqual(knownModelId, model.Id); // Invalid Model Id model = await modelsRegistry.GetModelAsync(unknownModelId, CancellationToken.None); Assert.IsNull(model); }
public async Task GetDefaultModelIdAsyncTest() { ITable table = Substitute.For <ITable>(); var modelsRegistry = new ModelsRegistry(table); // Default model not present in Models var knownModelId = Guid.NewGuid(); table.GetEntityAsync <ModelIdTableEntity>(ModelsRegistry.DefaultModelIdKeyName, CancellationToken.None, nameof(ModelIdTableEntity.ModelId)) .Returns(Task.FromResult(new ModelIdTableEntity(ModelsRegistry.DefaultModelIdKeyName, knownModelId))); var model = await modelsRegistry.GetDefaultModelAsync(CancellationToken.None); Assert.IsNull(model); // Default model present in Models table.GetEntityAsync <ModelTableEntity>(knownModelId.ToString(), CancellationToken.None, Arg.Any <string[]>()) .Returns(Task.FromResult(new ModelTableEntity(knownModelId))); model = await modelsRegistry.GetDefaultModelAsync(CancellationToken.None); Assert.IsNotNull(model); Assert.AreEqual(knownModelId, model.Id); }
public async Task ListModelsAsyncTest() { ITable table = Substitute.For <ITable>(); var modelsRegistry = new ModelsRegistry(table); table.ListEntitiesAsync <ModelTableEntity>(CancellationToken.None, Arg.Any <string[]>()) .Returns(Task.FromResult <IList <ModelTableEntity> >(new List <ModelTableEntity>())); var models = await modelsRegistry.ListModelsAsync(CancellationToken.None); Assert.IsNotNull(models); Assert.AreEqual(0, models.Count); table.ListEntitiesAsync <ModelTableEntity>(CancellationToken.None, Arg.Any <string[]>()) .Returns(Task.FromResult <IList <ModelTableEntity> >(new[] { new ModelTableEntity(Guid.NewGuid()), new ModelTableEntity(Guid.NewGuid()) })); models = await modelsRegistry.ListModelsAsync(CancellationToken.None); Assert.IsNotNull(models); Assert.AreEqual(2, models.Count); }
public async Task <IHttpActionResult> GetModel(CancellationToken cancellationToken, [FromUri] Guid?modelId) { if (!modelId.HasValue) { var message = $"{nameof(modelId)} is not valid"; Trace.TraceVerbose(message); return(BadRequest(message)); } // set the model id to context ContextManager.ModelId = modelId; Trace.TraceVerbose($"Trying to read the model '{modelId}' from the registry"); ModelsRegistry modelsRegistry = WebAppContext.ModelsRegistry; Model model = await modelsRegistry.GetModelAsync(modelId.Value, cancellationToken); if (model == null) { Trace.TraceInformation($"Model with id '{modelId}' does not exists."); return(NotFound()); } return(Ok(model)); }
public async Task DeleteModelIfExistsAsyncTest() { ITable table = Substitute.For <ITable>(); var modelsRegistry = new ModelsRegistry(table); var knownModelId = Guid.NewGuid(); table.DeleteEntityAsync <ModelTableEntity>(knownModelId.ToString(), CancellationToken.None) .Returns(Task.FromResult(true)); bool result = await modelsRegistry.DeleteModelIfExistsAsync(knownModelId, CancellationToken.None); Assert.IsTrue(result); await table.Received(1) .DeleteEntityAsync <ModelTableEntity>(Arg.Is <string>(id => Guid.Parse(id) == knownModelId), CancellationToken.None); var unknownModelId = Guid.NewGuid(); result = await modelsRegistry.DeleteModelIfExistsAsync(unknownModelId, CancellationToken.None); Assert.IsFalse(result); await table.Received(1).DeleteEntityAsync <ModelTableEntity>( Arg.Is <string>(id => Guid.Parse(id) == unknownModelId), CancellationToken.None); }
public async Task <IHttpActionResult> SetDefaultModel(CancellationToken cancellationToken, Guid?modelId) { if (!modelId.HasValue) { var message = $"{nameof(modelId)} is not valid"; Trace.TraceVerbose(message); return(BadRequest(message)); } // set the model id to context ContextManager.ModelId = modelId; Trace.TraceVerbose($"Trying to set '{modelId}' as the default model in the registry"); ModelsRegistry modelsRegistry = WebAppContext.ModelsRegistry; bool result = await modelsRegistry.SetDefaultModelIdAsync(modelId.Value, cancellationToken); if (!result) { Trace.TraceInformation($"Failed setting model with id '{modelId}' as the default model"); return(NotFound()); } return(Ok()); }
public async Task UpdateModelAsyncTest() { ITable table = Substitute.For <ITable>(); var modelsRegistry = new ModelsRegistry(table); table.MergeEntityAsync(Arg.Any <ModelTableEntity>(), CancellationToken.None).Returns(Task.FromResult(true)); var modelId = Guid.NewGuid(); // Scenario 1: All variables null. nothing should be overridden. await modelsRegistry.UpdateModelAsync(modelId, CancellationToken.None); await table.Received(1).MergeEntityAsync( Arg.Is <ModelTableEntity>( me => Guid.Parse(me.RowKey) == modelId && me.ModelStatus == null && me.StatusMessage == null && me.ModelStatistics == null), CancellationToken.None); // Scenario 2: Only status needs to be updated table.ClearReceivedCalls(); await modelsRegistry.UpdateModelAsync(modelId, CancellationToken.None, ModelStatus.InProgress); await table.Received(1).MergeEntityAsync( Arg.Is <ModelTableEntity>( me => Guid.Parse(me.RowKey) == modelId && me.ModelStatus == ModelStatus.InProgress.ToString() && me.StatusMessage == null && me.ModelStatistics == null), CancellationToken.None); // Scenario 3: Only status message needs to be updated table.ClearReceivedCalls(); await modelsRegistry.UpdateModelAsync(modelId, CancellationToken.None, null, "test"); await table.Received(1).MergeEntityAsync( Arg.Is <ModelTableEntity>( me => Guid.Parse(me.RowKey) == modelId && me.ModelStatus == null && me.StatusMessage == "test" && me.ModelStatistics == null), CancellationToken.None); // Scenario 4: Only statistics needs to be updated var modelStatistics = new ModelStatistics { NumberOfCatalogItems = 10 }; table.ClearReceivedCalls(); await modelsRegistry.UpdateModelAsync(modelId, CancellationToken.None, null, null, modelStatistics); await table.Received(1).MergeEntityAsync( Arg.Is <ModelTableEntity>( me => Guid.Parse(me.RowKey) == modelId && me.ModelStatus == null && me.StatusMessage == null && me.ModelStatistics == JsonConvert.SerializeObject(modelStatistics)), CancellationToken.None); // Scenario 5: everything needs to be updated table.ClearReceivedCalls(); await modelsRegistry.UpdateModelAsync(modelId, CancellationToken.None, ModelStatus.Completed, "test", modelStatistics); await table.Received(1).MergeEntityAsync( Arg.Is <ModelTableEntity>( me => Guid.Parse(me.RowKey) == modelId && me.ModelStatus == ModelStatus.Completed.ToString() && me.StatusMessage == "test" && me.ModelStatistics == JsonConvert.SerializeObject(modelStatistics)), CancellationToken.None); }
// ReSharper disable ParameterOnlyUsedForPreconditionCheck.Local private static void AssertPath <TModel>(PathInfo pi, string columnName, string query, KDPgValueType type) { Assert.Equal(type, pi.Expression.Type); Assert.Equal(query, pi.Expression.RawQuery.ToString()); Assert.Equal(ModelsRegistry.GetTable <TModel>().Columns.Find(x => x.Name == columnName), pi.Column); }
public RawQuery GetRawQuery() { if (IsEmpty) { return(RawQuery.Create("SELECT 0")); } RawQuery rq = new RawQuery(); var columns = _columns.Count == 0 ? AllColumnsWithoutAutoIncrement : _columns; rq.Append("INSERT INTO "); rq.AppendTableName(Table.Name, Table.Schema); rq.Append("("); rq.AppendColumnNames(columns.Select(x => x.columnDescriptor.Name)); if (_idColumn != null) { if (columns.Count > 0) { rq.Append(","); } rq.AppendColumnName(_idColumn.Name); } rq.Append(")"); rq.Append(" VALUES "); var first = true; foreach (var obj in _objects) { if (!first) { rq.Append(","); } rq.Append("("); for (int i = 0; i < columns.Count; i++) { var column = columns[i]; if (i > 0) { rq.Append(","); } if (column.subquery == null) { object val = ModelsRegistry.GetModelValueByColumn(obj, column.columnDescriptor); var npgValue = PgTypesConverter.ConvertToPgValue(column.columnDescriptor.Type, val); rq.Append(npgValue); } else { rq.AppendSurround(column.subquery.GetRawQuery()); } } if (_idColumn != null) { if (columns.Count > 0) { rq.Append(","); } rq.Append(ExpressionBuilders.CurrSeqValueOfTable(_idRefColumn).RawQuery); } rq.Append(")"); first = false; } if (_onInsertConflict == OnInsertConflict.DoNothing) { rq.Append(" ON CONFLICT DO NOTHING "); } if (_onInsertConflict == OnInsertConflict.DoUpdate) { rq.Append(" ON CONFLICT ("); var fields = new FieldListBuilder <TModel>(); _onInsertConflictUpdateFields(fields); first = true; foreach (var column in fields.Fields) { if (!first) { rq.Append(", "); } rq.AppendColumnName(column.Name); first = false; } rq.Append(") DO UPDATE SET "); var updateStatementsBuilder = new UpdateStatementsBuilder <TModel>(); _onInsertConflictUpdate(updateStatementsBuilder); first = true; foreach (var(column, typedExpression) in updateStatementsBuilder.UpdateParts) { if (!first) { rq.Append(", "); } rq.AppendColumnName(column.Name) .Append(" = ") .Append(typedExpression.RawQuery); first = false; } } if (TableModel.PrimaryKey != null) { rq.Append(" RETURNING "); rq.AppendColumnName(TableModel.PrimaryKey.Name); } rq.Append(";"); if (_outputVariable != null) { rq.Append(" SELECT "); rq.Append(ExpressionBuilders.SetConfigText(_outputVariable, ExpressionBuilders.LastVal(), true).RawQuery); } rq.SkipExplicitColumnTableNames(); return(rq); }