Пример #1
0
        public void WithColumnCombined()
        {
            var t1 = ModelsRegistry.GetTable <MyModel>();
            var t2 = ModelsRegistry.GetTable <MyModel2>();

            var rq1 = new RawQuery();

            rq1.AppendColumn(NodeVisitor.EvaluateFuncExpressionToColumn <MyModel>(x => x.Name), new RawQuery.TableNamePlaceholder(t1, "M1"));
            rq1.Append(" = 123");
            rq1.ApplyAlias("M1", "t1");

            var rq2 = new RawQuery();

            rq2.AppendColumn(NodeVisitor.EvaluateFuncExpressionToColumn <MyModel2>(x => x.Name1), new RawQuery.TableNamePlaceholder(t2, "M2"));
            rq2.Append(" = 456");
            rq2.ApplyAlias("M2", "t2");

            var rq = new RawQuery();

            rq.Append(rq1);
            rq.Append(", ");
            rq.Append(rq2);

            Utils.AssertRawQueryWithAliases(rq, @"t1.""name"" = 123, t2.name1 = 456");
        }
Пример #2
0
        public async Task <IHttpActionResult> GetModels(CancellationToken cancellationToken)
        {
            Trace.TraceVerbose("Listing all models in the registry");
            ModelsRegistry modelsRegistry = WebAppContext.ModelsRegistry;
            IList <Model>  models         = await modelsRegistry.ListModelsAsync(cancellationToken);

            return(Ok(models));
        }
Пример #3
0
        public static SelectFromBuilder FromCombinedExpression <TCombinedModel, TNewModel>(TablesList tablesList, Expression <Func <TCombinedModel, TNewModel> > prBody)
        {
            var builder = new SelectFromBuilder();

            builder.LeftJoinsExpressions = tablesList.JoinExpressions;

            var options = new EvaluationOptions();

            foreach (var tablePlaceholder in tablesList.Tables)
            {
                builder.AddTable(tablePlaceholder);
                // options.ParameterToTableAlias.Add(tablePlaceholder.Name, tablePlaceholder);
            }

            var tableToPlaceholder = tablesList.Tables.ToDictionary(x => x.Name);

            TypedExpression exp;

            switch (prBody.Body)
            {
            /* For:
             * .Select(x => new {
             *                  M1 = x.M1,
             *                  M2_name = x.M2.Name1,
             *                  M3_calc = x.M2.Id * 2,
             *              })
             */
            case NewExpression newExpression:
            {
                IEnumerable <Expression> args = newExpression.Arguments;

                var resultProcessor = new AnonymousTypeSelectResultProcessor <TNewModel>();
                builder.SelectResultProcessor = resultProcessor;

                foreach (var argExpression in args)
                {
                    // Member is Table (like M1 = x.M1)
                    if (argExpression is MemberExpression memberExpression && ModelsRegistry.IsTable(memberExpression.Type))
                    {
                        var tablePlaceholder = tableToPlaceholder[memberExpression.Member.Name];
                        var table            = tablePlaceholder.Table;

                        var tableTestRawQuery = new RawQuery();
                        tableTestRawQuery.AppendTable(tablePlaceholder);
                        tableTestRawQuery.Append(" IS NULL");
                        builder.AddSelectPart(tableTestRawQuery, KDPgValueTypeInstances.Boolean);

                        foreach (var column in table.Columns)
                        {
                            var rq = new RawQuery();
                            rq.AppendColumn(column, tablePlaceholder);
                            builder.AddSelectPart(rq, column.Type);
                        }

                        resultProcessor.AddModelEntry(table);
                    }
Пример #4
0
        public async Task <IHttpActionResult> TrainNewModel(CancellationToken cancellationToken, [FromBody] ModelParameters modelParameters)
        {
            // validate input
            if (modelParameters == null)
            {
                var message = $"Invalid format. Expected a valid '{nameof(ModelParameters)}' JSON";
                Trace.TraceVerbose(message);
                return(BadRequest(message));
            }

            if (!ModelState.IsValid)
            {
                return(BadRequest(ModelState));
            }

            ModelsRegistry modelsRegistry = WebAppContext.ModelsRegistry;

            Trace.TraceVerbose("Converting the model parameters to trainer settings, using default values where needed");
            var @default = ModelTrainingParameters.Default;
            var settings = new ModelTrainingParameters
            {
                BlobContainerName                     = modelParameters.BlobContainerName,
                CatalogFileRelativeLocation           = modelParameters.CatalogFileRelativeLocation?.Replace('\\', '/'),
                UsageFolderRelativeLocation           = modelParameters.UsageFolderRelativeLocation?.Replace('\\', '/'),
                EvaluationUsageFolderRelativeLocation = modelParameters.EvaluationUsageFolderRelativeLocation?.Replace('\\', '/'),
                SupportThreshold                = modelParameters.SupportThreshold ?? @default.SupportThreshold,
                CooccurrenceUnit                = modelParameters.CooccurrenceUnit ?? @default.CooccurrenceUnit,
                SimilarityFunction              = modelParameters.SimilarityFunction ?? @default.SimilarityFunction,
                EnableColdItemPlacement         = modelParameters.EnableColdItemPlacement ?? @default.EnableColdItemPlacement,
                EnableColdToColdRecommendations =
                    modelParameters.EnableColdToColdRecommendations ?? @default.EnableColdToColdRecommendations,
                EnableUserAffinity = modelParameters.EnableUserAffinity ?? @default.EnableUserAffinity,
                EnableUserToItemRecommendations =
                    modelParameters.EnableUserToItemRecommendations ?? @default.EnableUserToItemRecommendations,
                AllowSeedItemsInRecommendations =
                    modelParameters.AllowSeedItemsInRecommendations ?? @default.AllowSeedItemsInRecommendations,
                EnableBackfilling = modelParameters.EnableBackfilling ?? @default.EnableBackfilling,
                DecayPeriodInDays = modelParameters.DecayPeriodInDays ?? @default.DecayPeriodInDays
            };

            Trace.TraceInformation("Creating new model in registry");
            Model model = await modelsRegistry.CreateModelAsync(settings, modelParameters.Description, cancellationToken);

            Trace.TraceInformation($"Queueing a new train model message to the queue for model id {model.Id}");
            ModelQueueMessage modelQueueMessage = new ModelQueueMessage {
                ModelId = model.Id
            };
            await WebAppContext.TrainModelQueue.AddMessageAsync(modelQueueMessage, cancellationToken);

            // return the URL to the created model
            return(CreatedAtRoute(nameof(GetModel), new { modelId = model.Id }, model));
        }
Пример #5
0
        public async Task SetDefaultModelIdAsyncTest()
        {
            ITable table          = Substitute.For <ITable>();
            var    modelsRegistry = new ModelsRegistry(table);

            // Scenario 1: Invalid Model
            // Validate set api is not called
            table.GetEntityAsync <ModelTableEntity>(Arg.Any <string>(), CancellationToken.None, Arg.Any <string[]>())
            .Returns(Task.FromResult <ModelTableEntity>(null));
            table.InsertOrReplaceEntityAsync(Arg.Any <ModelTableEntity>(), CancellationToken.None)
            .Returns(Task.FromResult(true));
            var  unknownId = Guid.NewGuid();
            bool result    = await modelsRegistry.SetDefaultModelIdAsync(unknownId, CancellationToken.None);

            Assert.IsFalse(result);
            await table.DidNotReceive().InsertOrReplaceEntityAsync(Arg.Any <ModelIdTableEntity>(), Arg.Any <CancellationToken>());

            // Scenario 2: Valid Model but Status not Complete
            // Validate set api is not called
            var knownId = Guid.NewGuid();

            table.GetEntityAsync <ModelTableEntity>(knownId.ToString(), CancellationToken.None, Arg.Any <string[]>())
            .Returns(
                Task.FromResult(new ModelTableEntity(knownId)
            {
                ModelStatus = ModelStatus.InProgress.ToString()
            }));

            result = await modelsRegistry.SetDefaultModelIdAsync(knownId, CancellationToken.None);

            Assert.IsFalse(result);
            await table.DidNotReceive()
            .InsertOrReplaceEntityAsync(Arg.Any <ModelIdTableEntity>(), Arg.Any <CancellationToken>());

            // Scenario 3: Valid Model with Status as Complete
            // Validate set api is called
            table.GetEntityAsync <ModelTableEntity>(knownId.ToString(), CancellationToken.None, Arg.Any <string[]>())
            .Returns(
                Task.FromResult(new ModelTableEntity(knownId)
            {
                ModelStatus = ModelStatus.Completed.ToString()
            }));
            table.InsertOrReplaceEntityAsync(Arg.Any <ModelIdTableEntity>(), CancellationToken.None).Returns(Task.FromResult(true));
            result = await modelsRegistry.SetDefaultModelIdAsync(knownId, CancellationToken.None);

            Assert.IsTrue(result);
            await table.Received(1).InsertOrReplaceEntityAsync(
                Arg.Is <ModelIdTableEntity>(
                    entity => entity.RowKey == ModelsRegistry.DefaultModelIdKeyName && entity.ModelId == knownId),
                CancellationToken.None);
        }
Пример #6
0
        public async Task CreateModelAsyncTest()
        {
            ITable table          = Substitute.For <ITable>();
            var    modelsRegistry = new ModelsRegistry(table);

            table.InsertOrReplaceEntityAsync(Arg.Any <ModelIdTableEntity>(), CancellationToken.None).Returns(Task.FromResult(true));
            table.InsertEntityAsync(Arg.Any <ModelTableEntity>(), CancellationToken.None).Returns(Task.FromResult(true));

            Model model = await modelsRegistry.CreateModelAsync(ModelTrainingParameters.Default, null, CancellationToken.None);

            await table.Received(1).InsertEntityAsync(
                Arg.Is <ModelTableEntity>(me => Guid.Parse(me.RowKey) == model.Id && me.ModelStatus == ModelStatus.Created.ToString()),
                CancellationToken.None);
        }
Пример #7
0
        /// <summary>
        /// Creates  anew instance of the <see cref="WebJobLogic"/> class.
        /// </summary>
        /// <param name="modelsProvider">A models provider</param>
        /// <param name="modelsRegistry">A model registry</param>
        public WebJobLogic(ModelsProvider modelsProvider, ModelsRegistry modelsRegistry)
        {
            if (modelsProvider == null)
            {
                throw new ArgumentNullException(nameof(modelsProvider));
            }

            if (modelsRegistry == null)
            {
                throw new ArgumentNullException(nameof(modelsRegistry));
            }

            _modelsProvider = modelsProvider;
            _modelsRegistry = modelsRegistry;
        }
Пример #8
0
        public async Task <IHttpActionResult> GetDefaultModel(CancellationToken cancellationToken)
        {
            // set the model id to context
            ContextManager.ModelId = "default";

            Trace.TraceVerbose("Trying to read the default model from the registry");
            ModelsRegistry modelsRegistry = WebAppContext.ModelsRegistry;
            Model          defaultModel   = await modelsRegistry.GetDefaultModelAsync(cancellationToken);

            if (defaultModel == null)
            {
                Trace.TraceInformation("A default model is not defined");
                return(NotFound());
            }

            return(Ok(defaultModel));
        }
Пример #9
0
        public async Task ClearDefaultModelIdAsyncTest()
        {
            ITable table          = Substitute.For <ITable>();
            var    modelsRegistry = new ModelsRegistry(table);

            // Scenario 1: Model is set as default. Should return true after clearing.
            table.DeleteEntityAsync <ModelIdTableEntity>(ModelsRegistry.DefaultModelIdKeyName, CancellationToken.None)
            .Returns(Task.FromResult(true));
            Assert.IsTrue(await modelsRegistry.ClearDefaultModelIdAsync(CancellationToken.None));
            await table.Received(1).DeleteEntityAsync <ModelIdTableEntity>(ModelsRegistry.DefaultModelIdKeyName, CancellationToken.None);

            // Scenario 2: Table is unable to delete entity
            table.ClearReceivedCalls();
            table.DeleteEntityAsync <ModelIdTableEntity>(Arg.Any <string>(), CancellationToken.None)
            .Returns(Task.FromResult(false));
            Assert.IsFalse(await modelsRegistry.ClearDefaultModelIdAsync(CancellationToken.None));
            await table.Received(1).DeleteEntityAsync <ModelIdTableEntity>(ModelsRegistry.DefaultModelIdKeyName, CancellationToken.None);
        }
Пример #10
0
        public async Task GetModelAsyncTest()
        {
            ITable table          = Substitute.For <ITable>();
            var    modelsRegistry = new ModelsRegistry(table);

            var knownModelId   = Guid.NewGuid();
            var unknownModelId = Guid.NewGuid();

            table.GetEntityAsync <ModelTableEntity>(knownModelId.ToString(), CancellationToken.None, Arg.Any <string[]>())
            .Returns(Task.FromResult(new ModelTableEntity(knownModelId)));

            // Valid Model Id
            var model = await modelsRegistry.GetModelAsync(knownModelId, CancellationToken.None);

            Assert.IsNotNull(model);
            Assert.AreEqual(knownModelId, model.Id);

            // Invalid Model Id
            model = await modelsRegistry.GetModelAsync(unknownModelId, CancellationToken.None);

            Assert.IsNull(model);
        }
Пример #11
0
        public async Task GetDefaultModelIdAsyncTest()
        {
            ITable table          = Substitute.For <ITable>();
            var    modelsRegistry = new ModelsRegistry(table);

            // Default model not present in Models
            var knownModelId = Guid.NewGuid();

            table.GetEntityAsync <ModelIdTableEntity>(ModelsRegistry.DefaultModelIdKeyName, CancellationToken.None,
                                                      nameof(ModelIdTableEntity.ModelId))
            .Returns(Task.FromResult(new ModelIdTableEntity(ModelsRegistry.DefaultModelIdKeyName, knownModelId)));
            var model = await modelsRegistry.GetDefaultModelAsync(CancellationToken.None);

            Assert.IsNull(model);

            // Default model present in Models
            table.GetEntityAsync <ModelTableEntity>(knownModelId.ToString(), CancellationToken.None, Arg.Any <string[]>())
            .Returns(Task.FromResult(new ModelTableEntity(knownModelId)));
            model = await modelsRegistry.GetDefaultModelAsync(CancellationToken.None);

            Assert.IsNotNull(model);
            Assert.AreEqual(knownModelId, model.Id);
        }
Пример #12
0
        public async Task ListModelsAsyncTest()
        {
            ITable table          = Substitute.For <ITable>();
            var    modelsRegistry = new ModelsRegistry(table);

            table.ListEntitiesAsync <ModelTableEntity>(CancellationToken.None, Arg.Any <string[]>())
            .Returns(Task.FromResult <IList <ModelTableEntity> >(new List <ModelTableEntity>()));
            var models = await modelsRegistry.ListModelsAsync(CancellationToken.None);

            Assert.IsNotNull(models);
            Assert.AreEqual(0, models.Count);

            table.ListEntitiesAsync <ModelTableEntity>(CancellationToken.None, Arg.Any <string[]>())
            .Returns(Task.FromResult <IList <ModelTableEntity> >(new[]
            {
                new ModelTableEntity(Guid.NewGuid()),
                new ModelTableEntity(Guid.NewGuid())
            }));
            models = await modelsRegistry.ListModelsAsync(CancellationToken.None);

            Assert.IsNotNull(models);
            Assert.AreEqual(2, models.Count);
        }
Пример #13
0
        public async Task <IHttpActionResult> GetModel(CancellationToken cancellationToken, [FromUri] Guid?modelId)
        {
            if (!modelId.HasValue)
            {
                var message = $"{nameof(modelId)} is not valid";
                Trace.TraceVerbose(message);
                return(BadRequest(message));
            }

            // set the model id to context
            ContextManager.ModelId = modelId;

            Trace.TraceVerbose($"Trying to read the model '{modelId}' from the registry");
            ModelsRegistry modelsRegistry = WebAppContext.ModelsRegistry;
            Model          model          = await modelsRegistry.GetModelAsync(modelId.Value, cancellationToken);

            if (model == null)
            {
                Trace.TraceInformation($"Model with id '{modelId}' does not exists.");
                return(NotFound());
            }

            return(Ok(model));
        }
Пример #14
0
        public async Task DeleteModelIfExistsAsyncTest()
        {
            ITable table          = Substitute.For <ITable>();
            var    modelsRegistry = new ModelsRegistry(table);

            var knownModelId = Guid.NewGuid();

            table.DeleteEntityAsync <ModelTableEntity>(knownModelId.ToString(), CancellationToken.None)
            .Returns(Task.FromResult(true));

            bool result = await modelsRegistry.DeleteModelIfExistsAsync(knownModelId, CancellationToken.None);

            Assert.IsTrue(result);
            await table.Received(1)
            .DeleteEntityAsync <ModelTableEntity>(Arg.Is <string>(id => Guid.Parse(id) == knownModelId), CancellationToken.None);

            var unknownModelId = Guid.NewGuid();

            result = await modelsRegistry.DeleteModelIfExistsAsync(unknownModelId, CancellationToken.None);

            Assert.IsFalse(result);
            await table.Received(1).DeleteEntityAsync <ModelTableEntity>(
                Arg.Is <string>(id => Guid.Parse(id) == unknownModelId), CancellationToken.None);
        }
Пример #15
0
        public async Task <IHttpActionResult> SetDefaultModel(CancellationToken cancellationToken, Guid?modelId)
        {
            if (!modelId.HasValue)
            {
                var message = $"{nameof(modelId)} is not valid";
                Trace.TraceVerbose(message);
                return(BadRequest(message));
            }

            // set the model id to context
            ContextManager.ModelId = modelId;

            Trace.TraceVerbose($"Trying to set '{modelId}' as the default model in the registry");
            ModelsRegistry modelsRegistry = WebAppContext.ModelsRegistry;
            bool           result         = await modelsRegistry.SetDefaultModelIdAsync(modelId.Value, cancellationToken);

            if (!result)
            {
                Trace.TraceInformation($"Failed setting model with id '{modelId}' as the default model");
                return(NotFound());
            }

            return(Ok());
        }
Пример #16
0
        public async Task UpdateModelAsyncTest()
        {
            ITable table          = Substitute.For <ITable>();
            var    modelsRegistry = new ModelsRegistry(table);

            table.MergeEntityAsync(Arg.Any <ModelTableEntity>(), CancellationToken.None).Returns(Task.FromResult(true));

            var modelId = Guid.NewGuid();

            // Scenario 1: All variables null. nothing should be overridden.
            await modelsRegistry.UpdateModelAsync(modelId, CancellationToken.None);

            await table.Received(1).MergeEntityAsync(
                Arg.Is <ModelTableEntity>(
                    me => Guid.Parse(me.RowKey) == modelId && me.ModelStatus == null && me.StatusMessage == null && me.ModelStatistics == null),
                CancellationToken.None);

            // Scenario 2: Only status needs to be updated
            table.ClearReceivedCalls();
            await modelsRegistry.UpdateModelAsync(modelId, CancellationToken.None, ModelStatus.InProgress);

            await table.Received(1).MergeEntityAsync(
                Arg.Is <ModelTableEntity>(
                    me =>
                    Guid.Parse(me.RowKey) == modelId && me.ModelStatus == ModelStatus.InProgress.ToString() &&
                    me.StatusMessage == null && me.ModelStatistics == null), CancellationToken.None);

            // Scenario 3: Only status message needs to be updated
            table.ClearReceivedCalls();
            await modelsRegistry.UpdateModelAsync(modelId, CancellationToken.None, null, "test");

            await table.Received(1).MergeEntityAsync(
                Arg.Is <ModelTableEntity>(
                    me => Guid.Parse(me.RowKey) == modelId && me.ModelStatus == null && me.StatusMessage == "test" && me.ModelStatistics == null),
                CancellationToken.None);

            // Scenario 4: Only statistics needs to be updated
            var modelStatistics = new ModelStatistics
            {
                NumberOfCatalogItems = 10
            };

            table.ClearReceivedCalls();
            await modelsRegistry.UpdateModelAsync(modelId, CancellationToken.None, null, null, modelStatistics);

            await table.Received(1).MergeEntityAsync(
                Arg.Is <ModelTableEntity>(
                    me =>
                    Guid.Parse(me.RowKey) == modelId && me.ModelStatus == null && me.StatusMessage == null &&
                    me.ModelStatistics == JsonConvert.SerializeObject(modelStatistics)),
                CancellationToken.None);

            // Scenario 5: everything needs to be updated
            table.ClearReceivedCalls();
            await modelsRegistry.UpdateModelAsync(modelId, CancellationToken.None, ModelStatus.Completed, "test", modelStatistics);

            await table.Received(1).MergeEntityAsync(
                Arg.Is <ModelTableEntity>(
                    me =>
                    Guid.Parse(me.RowKey) == modelId && me.ModelStatus == ModelStatus.Completed.ToString() &&
                    me.StatusMessage == "test" && me.ModelStatistics == JsonConvert.SerializeObject(modelStatistics)),
                CancellationToken.None);
        }
Пример #17
0
 // ReSharper disable ParameterOnlyUsedForPreconditionCheck.Local
 private static void AssertPath <TModel>(PathInfo pi, string columnName, string query, KDPgValueType type)
 {
     Assert.Equal(type, pi.Expression.Type);
     Assert.Equal(query, pi.Expression.RawQuery.ToString());
     Assert.Equal(ModelsRegistry.GetTable <TModel>().Columns.Find(x => x.Name == columnName), pi.Column);
 }
Пример #18
0
        public RawQuery GetRawQuery()
        {
            if (IsEmpty)
            {
                return(RawQuery.Create("SELECT 0"));
            }

            RawQuery rq = new RawQuery();

            var columns = _columns.Count == 0 ? AllColumnsWithoutAutoIncrement : _columns;

            rq.Append("INSERT INTO ");

            rq.AppendTableName(Table.Name, Table.Schema);

            rq.Append("(");
            rq.AppendColumnNames(columns.Select(x => x.columnDescriptor.Name));
            if (_idColumn != null)
            {
                if (columns.Count > 0)
                {
                    rq.Append(",");
                }
                rq.AppendColumnName(_idColumn.Name);
            }

            rq.Append(")");

            rq.Append(" VALUES ");

            var first = true;

            foreach (var obj in _objects)
            {
                if (!first)
                {
                    rq.Append(",");
                }
                rq.Append("(");

                for (int i = 0; i < columns.Count; i++)
                {
                    var column = columns[i];

                    if (i > 0)
                    {
                        rq.Append(",");
                    }

                    if (column.subquery == null)
                    {
                        object val      = ModelsRegistry.GetModelValueByColumn(obj, column.columnDescriptor);
                        var    npgValue = PgTypesConverter.ConvertToPgValue(column.columnDescriptor.Type, val);
                        rq.Append(npgValue);
                    }
                    else
                    {
                        rq.AppendSurround(column.subquery.GetRawQuery());
                    }
                }

                if (_idColumn != null)
                {
                    if (columns.Count > 0)
                    {
                        rq.Append(",");
                    }

                    rq.Append(ExpressionBuilders.CurrSeqValueOfTable(_idRefColumn).RawQuery);
                }

                rq.Append(")");
                first = false;
            }

            if (_onInsertConflict == OnInsertConflict.DoNothing)
            {
                rq.Append(" ON CONFLICT DO NOTHING ");
            }

            if (_onInsertConflict == OnInsertConflict.DoUpdate)
            {
                rq.Append(" ON CONFLICT (");

                var fields = new FieldListBuilder <TModel>();
                _onInsertConflictUpdateFields(fields);
                first = true;
                foreach (var column in fields.Fields)
                {
                    if (!first)
                    {
                        rq.Append(", ");
                    }

                    rq.AppendColumnName(column.Name);

                    first = false;
                }

                rq.Append(") DO UPDATE SET ");

                var updateStatementsBuilder = new UpdateStatementsBuilder <TModel>();
                _onInsertConflictUpdate(updateStatementsBuilder);

                first = true;
                foreach (var(column, typedExpression) in updateStatementsBuilder.UpdateParts)
                {
                    if (!first)
                    {
                        rq.Append(", ");
                    }

                    rq.AppendColumnName(column.Name)
                    .Append(" = ")
                    .Append(typedExpression.RawQuery);

                    first = false;
                }
            }

            if (TableModel.PrimaryKey != null)
            {
                rq.Append(" RETURNING ");
                rq.AppendColumnName(TableModel.PrimaryKey.Name);
            }

            rq.Append(";");

            if (_outputVariable != null)
            {
                rq.Append(" SELECT ");
                rq.Append(ExpressionBuilders.SetConfigText(_outputVariable, ExpressionBuilders.LastVal(), true).RawQuery);
            }

            rq.SkipExplicitColumnTableNames();
            return(rq);
        }