Пример #1
0
        public async Task <YEngine> SaveEngineAsync(YEngine engine)
        {
            try
            {
                using CosmosClient client = new CosmosClient(accountEndpoint, accountKey, clientOptions);
                var container = client.GetContainer(databaseName, containerName);

                engine.UpdateDate = DateTime.Now;

                if (!engine.RequestDate.HasValue || engine.RequestDate == DateTime.MinValue)
                {
                    engine.RequestDate = DateTime.Now;
                }

                ItemResponse <YEngine> response = await container.UpsertItemAsync(
                    engine, enginePartitionKey).ConfigureAwait(false);

                return(engine);
            }
            catch (Exception ex)
            {
                Debug.WriteLine(ex);
                return(null);
            }
        }
Пример #2
0
        public Job GetProcessingJob(YEngine engine)
        {
            var monitoring = JobStorage.Current.GetMonitoringApi();
            var jobs       = monitoring.ProcessingJobs(0, 1000);

            if (jobs == null || jobs.Count <= 0)
            {
                return(null);
            }

            var job = jobs.Select(kv => kv.Value.Job).FirstOrDefault(j =>
            {
                if (j.Args == null || j.Args.Count <= 0)
                {
                    return(false);
                }

                if (!(j.Args[0] is YEngine engineJob))
                {
                    return(false);
                }

                if (engineJob.Id == engine.Id)
                {
                    return(true);
                }

                return(false);
            });

            return(job);
        }
Пример #3
0
        private List <Guid> GetNotifiersGroup(YEngine engine, Guid?callerUserId = default, bool addMembers = false)
        {
            var allUsersToNotify = engine.Owners?.ToList();


            if (addMembers && allUsersToNotify != null && engine.Members != null && engine.Members.Count > 0)
            {
                allUsersToNotify = allUsersToNotify.Union(engine.Members).Distinct().ToList();
            }

            List <Guid> groupUsers = null;

            if (allUsersToNotify != null && allUsersToNotify.Count > 0)
            {
                groupUsers = allUsersToNotify.Select(u => u.Id).ToList();
            }

            if (callerUserId.HasValue)
            {
                if (groupUsers == null)
                {
                    groupUsers = new List <Guid>();
                }

                if (!groupUsers.Any(u => u == callerUserId.Value))
                {
                    groupUsers.Add(callerUserId.Value);
                }
            }

            return(groupUsers);
        }
Пример #4
0
        public async Task SendRefreshNotificationsAsync(YEngine engine, Guid?callerUserId = default, bool addMembers = false)
        {
            var groupUsers = GetNotifiersGroup(engine, callerUserId, addMembers);

            if (groupUsers != null && groupUsers.Count > 0)
            {
                await SendRefreshNotificationsAsync(groupUsers).ConfigureAwait(false);
            }
        }
Пример #5
0
        public Task <YJsonResult <YEngine> > SaveEngineAsync(Guid?id, [FromBody] YEngine engine)
        {
            return(YExecuteAsync(async() =>
            {
                if (!id.HasValue)
                {
                    throw new ArgumentNullException(nameof(id));
                }

                var response = await this.client.ProcessRequestApiAsync <YEngine>($"api/Engines/{id}", null, engine, HttpMethod.Put).ConfigureAwait(false);

                return response.Value;
            }));
        }
Пример #6
0
        public async Task <ActionResult <YDeploymentStatePayload> > DeleteEngineAsync(Guid id)
        {
            HttpContext.VerifyUserHasAnyAcceptedScope(scopeRequiredByApi);

            if (!this.User.IsInRole("Admin"))
            {
                return(new UnauthorizedObjectResult("You should be admin to remove an engine and all its resources."));
            }

            var userObjectId = this.User.GetObjectId();

            if (string.IsNullOrEmpty(userObjectId))
            {
                return(new UnauthorizedObjectResult("User unknown"));
            }

            var userId = new Guid(userObjectId);

            YEngine engine = await this.engineProvider.GetEngineAsync(id);

            if (engine == null)
            {
                return(new NotFoundObjectResult($"No engine found with Id {id}"));
            }

            var health = await this.notificationsService.IsServiceHealthyAsync();

            if (!health)
            {
                throw new Exception("Azure SignalR Service is not healthy");
            }


            var jobId = BackgroundJob.Enqueue(() => enginesService.DeleteEngineDeploymentAsync(engine, userId, default));

            var deployingState = new YDeploymentStatePayload(YDeploymentStatePayloadState.Deploying)
            {
                Id      = id,
                Message = $"Removing engine deployment started. Background Job Id : {jobId}."
            };

            return(deployingState);
        }
Пример #7
0
        private async Task PostMetric2Async(YEngine engine, YMetricPayload2 payload, string instrumentationKey)
        {
            using TelemetryConfiguration configuration = TelemetryConfiguration.CreateDefault();

            configuration.InstrumentationKey = instrumentationKey;

            var telemetryClient = new TelemetryClient(configuration);

            var telemetry = new MetricTelemetry(payload.Namespace, payload.Name, payload.Count, payload.Sum, payload.Min, payload.Max, 0);

            foreach (var d in payload.Dimensions)
            {
                telemetry.Properties.Add(d.Name, d.Value);
            }

            telemetryClient.TrackMetric(telemetry);

            telemetryClient.Flush();
        }
Пример #8
0
        public async Task <ActionResult <YDeploymentStatePayload> > DeployEngineAsync(Guid id, [FromBody] YEngine engine)
        {
            HttpContext.VerifyUserHasAnyAcceptedScope(scopeRequiredByApi);

            if (!this.User.IsInRole("Admin"))
            {
                return(new UnauthorizedObjectResult("You should be admin to make a deployment"));
            }

            var userObjectId = this.User.GetObjectId();

            if (string.IsNullOrEmpty(userObjectId))
            {
                return(new UnauthorizedObjectResult("User unknown"));
            }

            var userId = new Guid(userObjectId);

            engine = await this.engineProvider.GetEngineAsync(id).ConfigureAwait(false);

            var settings = await this.settingsProvider.GetSettingsAsync().ConfigureAwait(false);

            if (settings != null)
            {
                if (string.IsNullOrEmpty(engine.ResourceGroupName))
                {
                    string prefix = settings.FirstOrDefault(s => s.Name == "ResourceGroupPrefix")?.Value ?? "rg";
                    string suffix = settings.FirstOrDefault(s => s.Name == "ResourceGroupSuffix")?.Value;

                    string name = $"{prefix}{engine.EngineName}{suffix}";
                    engine.ResourceGroupName = name;
                }
                if (string.IsNullOrEmpty(engine.ClusterName))
                {
                    string prefix = settings.FirstOrDefault(s => s.Name == "DatabricksWorkspacePrefix")?.Value ?? "dw";
                    string suffix = settings.FirstOrDefault(s => s.Name == "DatabricksWorkspaceSuffix")?.Value;

                    string name = $"{prefix}{engine.EngineName}{suffix}";
                    engine.ClusterName = name;
                }
                if (string.IsNullOrEmpty(engine.FactoryName))
                {
                    string prefix = settings.FirstOrDefault(s => s.Name == "DataFactoryPrefix")?.Value ?? "df";
                    string suffix = settings.FirstOrDefault(s => s.Name == "DataFactorySuffix")?.Value;

                    string name = $"{prefix}{engine.EngineName}{suffix}";
                    engine.FactoryName = name;
                }
                if (string.IsNullOrEmpty(engine.KeyVaultName))
                {
                    string prefix = settings.FirstOrDefault(s => s.Name == "KeyVaultPrefix")?.Value ?? "kv";
                    string suffix = settings.FirstOrDefault(s => s.Name == "KeyVaultSuffix")?.Value;

                    string name = $"{prefix}{engine.EngineName}{suffix}";
                    engine.KeyVaultName = name;
                }
                if (string.IsNullOrEmpty(engine.StorageName))
                {
                    string prefix = settings.FirstOrDefault(s => s.Name == "StoragePrefix")?.Value ?? "stor";
                    string suffix = settings.FirstOrDefault(s => s.Name == "StorageSuffix")?.Value;

                    string name = $"{prefix}{engine.EngineName}{suffix}";
                    engine.StorageName = name;
                }
                if (string.IsNullOrEmpty(engine.AppInsightsName))
                {
                    string prefix = settings.FirstOrDefault(s => s.Name == "AppInsightsPrefix")?.Value ?? "ai";
                    string suffix = settings.FirstOrDefault(s => s.Name == "AppInsightsSuffix")?.Value;

                    string name = $"{prefix}{engine.EngineName}{suffix}";
                    engine.AppInsightsName = name;
                }

                if (string.IsNullOrEmpty(engine.Location))
                {
                    string location = settings.FirstOrDefault(s => s.Name == "DefaultLocation")?.Value ?? "northeurope";
                    engine.Location = location;
                }
            }

            engine.ResourceGroupName.EnsureStringIsLetterOrDigit();
            engine.ClusterName.EnsureStringIsLetterOrDigit();
            engine.FactoryName.EnsureStringIsLetterOrDigit();
            engine.StorageName.EnsureStringIsLetterOrDigit();
            engine.AppInsightsName.EnsureStringIsLetterOrDigit();
            engine.KeyVaultName.EnsureStringIsLetterOrDigit();

            var job = this.hangFireService.GetProcessingJob(engine);

            if (job != null)
            {
                return(new YDeploymentStatePayload(YDeploymentStatePayloadState.Deploying)
                {
                    Id = id,
                    Message = $"Background process indicate your engine <strong>{engine.EngineName}</strong> has a deployment already in progress.<br />You'll receive a notification when the deployment is completed."
                });
            }

            var health = await this.notificationsService.IsServiceHealthyAsync();

            if (!health)
            {
                throw new Exception("Azure SignalR Service is not healthy");
            }

            var jobId = BackgroundJob.Enqueue(() => enginesService.CreateEngineDeploymentAsync(engine, userId, default));

            var deployingState = new YDeploymentStatePayload(YDeploymentStatePayloadState.Deploying)
            {
                Id      = id,
                Message = $"Deploying start. Background Job Id : {jobId}."
            };

            return(deployingState);
        }
Пример #9
0
        public async Task <ActionResult <YEngine> > SaveEngineAsync(Guid id, [FromBody] YEngine engine)
        {
            HttpContext.VerifyUserHasAnyAcceptedScope(scopeRequiredByApi);

            var userObjectId = this.User.GetObjectId();

            if (id != engine.Id)
            {
                return(new UnprocessableEntityObjectResult("id and entity differs"));
            }

            if (string.IsNullOrEmpty(userObjectId))
            {
                return(new UnauthorizedObjectResult("User unknown"));
            }

            var userId = new Guid(userObjectId);

            if (!string.IsNullOrEmpty(engine.EngineName))
            {
                var regex = new Regex(@"^[A-Za-z0-9]+(?:-[A-Za-z0-9]+)*$");

                if (!regex.IsMatch(engine.EngineName))
                {
                    throw new Exception($"Engine name {engine.EngineName} is incorrect. The regex used to validate the name is {regex}");
                }

                if (engine.EngineName.Length < 5 || engine.EngineName.Length > 10)
                {
                    throw new Exception("The engine name needs to be between 5 and 10 characters long.");
                }
            }

            if (!string.IsNullOrEmpty(engine.ResourceGroupName))
            {
                var regex = new Regex(@"^[-\w\._\(\)]+$");

                if (!regex.IsMatch(engine.ResourceGroupName))
                {
                    throw new Exception($"Resource group name {engine.ResourceGroupName} is incorrect");
                }
            }

            if (!string.IsNullOrEmpty(engine.FactoryName))
            {
                var regex = new Regex(@"^[A-Za-z0-9]+(?:-[A-Za-z0-9]+)*$");

                if (!regex.IsMatch(engine.FactoryName))
                {
                    throw new Exception($"Factory name {engine.FactoryName} is incorrect");
                }
            }

            if (!string.IsNullOrEmpty(engine.ClusterName))
            {
                var regex = new Regex(@"^[A-Za-z0-9]+(?:-[A-Za-z0-9]+)*$");

                if (!regex.IsMatch(engine.ClusterName))
                {
                    throw new Exception($"Cluster name {engine.ClusterName} is incorrect");
                }
            }
            if (!string.IsNullOrEmpty(engine.KeyVaultName))
            {
                var regex = new Regex(@"^[a-zA-Z0-9-]{3,24}$");

                if (!regex.IsMatch(engine.KeyVaultName))
                {
                    throw new Exception($"KeyVault name {engine.KeyVaultName} is incorrect");
                }
            }
            if (!string.IsNullOrEmpty(engine.StorageName))
            {
                var regex = new Regex(@"^[a-z0-9-]{3,24}$");

                if (!regex.IsMatch(engine.StorageName))
                {
                    throw new Exception($"Storage account name {engine.StorageName} is incorrect");
                }
            }
            if (!string.IsNullOrEmpty(engine.AppInsightsName))
            {
                var regex = new Regex(@"^[a-z0-9-]{3,24}$");

                if (!regex.IsMatch(engine.AppInsightsName))
                {
                    throw new Exception($"AppInsights name {engine.AppInsightsName} is incorrect");
                }
            }

            if (engine.OwnerEmails != null && engine.OwnerEmails.Count > 0)
            {
                //Lookup IDs for the provided owner email addresses to complete the request.
            }

            var existingEngine = await this.engineProvider.GetEngineByNameAsync(engine.EngineName);

            if (existingEngine != null && existingEngine.Id != engine.Id)
            {
                throw new Exception($"The Engine {engine.EngineName} already exists. Please choose another name");
            }

            var engineSaved = await this.engineProvider.SaveEngineAsync(engine, userId);

            if (!string.IsNullOrEmpty(engineSaved.AdminComments) && engineSaved.Status == YEngineStatus.NeedMoreInfos)
            {
                // Only send notification to owners
                var allUsersToNotify = engine.Owners?.Select(u => u.Id);

                if (allUsersToNotify != null)
                {
                    await notificationsService.CreateNotificationsDeploymentDoneAsync(engine.Id, "Admin comments",
                                                                                      $"You have engine <a href='/Engines/Details/{engine.Id}' >admin comments</a> on engine <b>{engine.EngineName}</b>",
                                                                                      allUsersToNotify, $"/Engines/Details/{engine.Id}").ConfigureAwait(false);
                }
            }

            return(engineSaved);
        }
Пример #10
0
        public async Task <ActionResult <YDeploymentStatePayload> > DeployEngineAsync(Guid id, [FromBody] YEngine engine)
        {
            HttpContext.VerifyUserHasAnyAcceptedScope(scopeRequiredByApi);

            if (!this.User.IsInRole("Admin"))
            {
                return(new UnauthorizedObjectResult("You should be admin to make a deployment"));
            }

            var userObjectId = this.User.GetObjectId();

            if (string.IsNullOrEmpty(userObjectId))
            {
                return(new UnauthorizedObjectResult("User unknown"));
            }

            var userId = new Guid(userObjectId);

            engine = await this.engineProvider.GetEngineAsync(id).ConfigureAwait(false);

            engine.ResourceGroupName.EnsureStringIsLetterOrDigit();
            engine.ClusterName.EnsureStringIsLetterOrDigit();
            engine.FactoryName.EnsureStringIsLetterOrDigit();
            engine.StorageName.EnsureStringIsLetterOrDigit();
            engine.AppInsightsName.EnsureStringIsLetterOrDigit();
            engine.KeyVaultName.EnsureStringIsLetterOrDigit();

            var job = this.hangFireService.GetProcessingJob(engine);

            if (job != null)
            {
                return(new YDeploymentStatePayload(YDeploymentStatePayloadState.Deploying)
                {
                    Id = id,
                    Message = $"Background process indicate your engine <strong>{engine.EngineName}</strong> has a deployment already in progress.<br />You'll receive a notification when the deployment is completed."
                });
            }

            var health = await this.notificationsService.IsServiceHealthyAsync();

            if (!health)
            {
                throw new Exception("Azure SignalR Service is not healthy");
            }

            var jobId = BackgroundJob.Enqueue(() => enginesService.CreateEngineDeploymentAsync(engine, userId, default));

            var deployingState = new YDeploymentStatePayload(YDeploymentStatePayloadState.Deploying)
            {
                Id      = id,
                Message = $"Deploying start. Background Job Id : {jobId}."
            };

            return(deployingState);
        }
Пример #11
0
 public EngineView(YEngine engineDeployment)
 {
     this.IsNew  = false;
     this.Engine = engineDeployment;
 }
Пример #12
0
        public async Task CreateNotificationsDeploymentDoneAsync(string title, string message, YEngine engine, Guid?callerUserId = default, string url = default, bool addMembers = false)
        {
            var groupUsers = GetNotifiersGroup(engine, callerUserId, addMembers);

            if (groupUsers != null && groupUsers.Count > 0)
            {
                await CreateNotificationsDeploymentDoneAsync(engine.Id, title, message, groupUsers, url).ConfigureAwait(false);
            }
        }
Пример #13
0
        public async Task SendNotificationAsync(string method, YDeploymentStatePayloadState state, YEngine engine, string message, Guid?callerUserId = default, object resource = null, bool addMembers = false)
        {
            var groupUsers = GetNotifiersGroup(engine, callerUserId, addMembers);

            List <string> groups = null;

            if (groupUsers != null)
            {
                groups = groupUsers.Select(guid => $"{engine.Id}-{guid}").ToList();
            }

            if (groups == null || groups.Count <= 0)
            {
                return;
            }

            var hubContext = await serviceManager.CreateHubContextAsync("JobHub").ConfigureAwait(false);

            var deployState = new YDeploymentStatePayload(state)
            {
                Id      = engine.Id,
                Message = message
            };

            if (resource != null)
            {
                await hubContext.Clients.Groups(groups).SendAsync(method, deployState, resource, default).ConfigureAwait(false);
            }
            else
            {
                await hubContext.Clients.Groups(groups).SendAsync(method, deployState, default).ConfigureAwait(false);
            }
        }
Пример #14
0
        private async Task CreatePipelineAsync(YEngine engine, YEntity entity)
        {
            // -------------------------
            // PIPELINE


            string version = entity.Version.Replace(".", "_");


            if (string.IsNullOrEmpty(version))
            {
                version = "v1";
            }

            // try to create a Copy Pipeline
            string pipelineName = $"{entity.DataSourceName.ToLower()}_{entity.Name.ToLower()}_{version.ToLower()}";


            var pipelinePathUri = $"/subscriptions/{options.SubscriptionId}" +
                                  $"/resourceGroups/{engine.ResourceGroupName}/providers/Microsoft.DataFactory" +
                                  $"/factories/{engine.FactoryName}/pipelines/{pipelineName}";

            var pipelineQuery = $"api-version={DataFactoryApiVersion}";


            var copyPipeline = new YPipeline
            {
                Name = pipelineName
            };

            // Copy Pipeline

            var copyActivity = new YPipelineActivity
            {
                Name = "Loading",
                Type = "Copy"
            };

            var source = new YPipelineSource
            {
                Type = entity.EntityType switch
                {
                    YEntityType.DelimitedText => "DelimitedTextSource",
                    YEntityType.AzureSqlTable => "AzureSqlSource",
                    YEntityType.Parquet => "ParquetSource",
                    _ => "DelimitedTextSource",
                }
            };

            if (entity.EntityType == YEntityType.Parquet || entity.EntityType == YEntityType.DelimitedText)
            {
                source.StoreSettings                  = new YPipelineStoreSettings();
                source.StoreSettings.Recursive        = true;
                source.StoreSettings.WildcardFileName = "*";
                source.StoreSettings.Type             = "AzureBlobStorageReadSettings";
            }
            else if (entity.EntityType == YEntityType.AzureSqlTable)
            {
                // only creates if we have the entity supporting it AND request by the method
                if (entity.Mode == "Delta")
                {
                    var sqlEntity = YEntityFactory.GetTypedEntity(entity) as YEntityAzureSqlTable;

                    source.SqlReaderQuery = new YValueType
                    {
                        Type  = "Expression",
                        Value = $"Declare @startDate Datetime = '@{{formatDateTime(pipeline().parameters.windowStart, 'yyyy-MM-dd HH:mm')}}'; " +
                                $"Declare @fullLoad boolean = @{{pipeline().parameters.fullLoad}}; " +
                                $"Select * From [{sqlEntity.Schema}].[{sqlEntity.Table}] " +
                                $"Where ([ModifiedDate] >= @startDate And @fullLoad = 0) OR (@fullLoad = 1)"
                    };
                }
                source.PartitionOption = "none";
            }
            else
            {
                source.PartitionOption = "none";
            }

            copyActivity.TypeProperties.Add("source", JObject.FromObject(source));

            var sink = new YPipelineSink {
                Type = "ParquetSink"
            };

            sink.StoreSettings.Type  = "AzureBlobFSWriteSettings";
            sink.FormatSettings.Type = "ParquetWriteSettings";
            copyActivity.TypeProperties.Add("sink", JObject.FromObject(sink));

            copyActivity.TypeProperties.Add("enableStaging", false);


            copyActivity.Inputs = new List <YPipelineReference>();
            copyActivity.Inputs.Add(new YPipelineReference
            {
                ReferenceName = entity.Name,
                Type          = "DatasetReference"
            });

            var output = new YPipelineOutput
            {
                ReferenceName = "destinationOutput",
                Type          = "DatasetReference",
            };

            output.Parameters.FileSystem.Type  = "Expression";
            output.Parameters.FileSystem.Value = "@pipeline().parameters.destinationContainer";
            output.Parameters.FolderPath.Type  = "Expression";
            output.Parameters.FolderPath.Value = "@{pipeline().parameters.destinationFolderPath}/@{formatDateTime(pipeline().parameters.windowStart,'yyyy')}/@{formatDateTime(pipeline().parameters.windowStart,'MM')}/@{formatDateTime(pipeline().parameters.windowStart,'dd')}/@{formatDateTime(pipeline().parameters.windowStart,'HH')}";

            copyActivity.Outputs = new List <YPipelineOutput>();
            copyActivity.Outputs.Add(output);

            copyPipeline.Properties.Activities.Add(copyActivity);

            // databricks

            var dbricksActivity = new YPipelineActivity
            {
                Name = "Transform",
                Type = "DatabricksNotebook"
            };

            var dependOn = new YPipelineDependsOn {
                Activity = "Loading"
            };

            dependOn.DependencyConditions.Add("Succeeded");
            dbricksActivity.DependsOn.Add(dependOn);

            dbricksActivity.LinkedServiceName = new YPipelineReference {
                ReferenceName = $"dsDatabricks-{engine.ClusterName}", Type = "LinkedServiceReference"
            };

            dbricksActivity.TypeProperties.Add("notebookPath", "/Shared/main");
            dbricksActivity.TypeProperties.Add("baseParameters", new JObject {
                { "entityName", new JObject {
                      { "value", "@{pipeline().parameters.entityName}" }, { "type", "Expression" }
                  } },
                { "inputPath", new JObject {
                      { "value", "@{concat(pipeline().parameters.destinationFolderPath, '/', formatDateTime(pipeline().parameters.windowStart,'yyyy'), '/', formatDateTime(pipeline().parameters.windowStart,'MM'), '/', formatDateTime(pipeline().parameters.windowStart,'dd'), '/', formatDateTime(pipeline().parameters.windowStart,'HH'))}" }, { "type", "Expression" }
                  } },
                { "outputPath", new JObject {
                      { "value", "@{pipeline().parameters.deltaFolderPath}" }, { "type", "Expression" }
                  } },
                { "inputContainer", new JObject {
                      { "value", "@{pipeline().parameters.destinationContainer}" }, { "type", "Expression" }
                  } },
                { "outputContainer", new JObject {
                      { "value", "@{pipeline().parameters.deltaContainer}" }, { "type", "Expression" }
                  } },
                { "dataSourceName", new JObject {
                      { "value", "@{pipeline().parameters.dataSourceName}" }, { "type", "Expression" }
                  } },
                { "version", new JObject {
                      { "value", "@{pipeline().parameters.version}" }, { "type", "Expression" }
                  } }
            });

            copyPipeline.Properties.Activities.Add(dbricksActivity);


            copyPipeline.Properties.Parameters.Add("windowStart", JObject.FromObject(new YPipelineParameter {
                DefaultValue = DateTime.Now
            }));
            copyPipeline.Properties.Parameters.Add("fullLoad", JObject.FromObject(new YPipelineParameter {
                DefaultValue = "1"
            }));
            copyPipeline.Properties.Parameters.Add("destinationContainer", JObject.FromObject(new YPipelineParameter {
                DefaultValue = "bronze"
            }));
            copyPipeline.Properties.Parameters.Add("destinationFolderPath", JObject.FromObject(new YPipelineParameter {
                DefaultValue = $"{entity.DataSourceName}/{entity.Name}/{version}"
            }));
            copyPipeline.Properties.Parameters.Add("deltaContainer", JObject.FromObject(new YPipelineParameter {
                DefaultValue = "silver"
            }));
            copyPipeline.Properties.Parameters.Add("deltaFolderPath", JObject.FromObject(new YPipelineParameter {
                DefaultValue = $"{entity.DataSourceName}/{entity.Name}/{version}"
            }));
            copyPipeline.Properties.Parameters.Add("engineId", JObject.FromObject(new YPipelineParameter {
                DefaultValue = engine.Id
            }));
            copyPipeline.Properties.Parameters.Add("entityName", JObject.FromObject(new YPipelineParameter {
                DefaultValue = entity.Name
            }));
            copyPipeline.Properties.Parameters.Add("dataSourceName", JObject.FromObject(new YPipelineParameter {
                DefaultValue = entity.DataSourceName
            }));
            copyPipeline.Properties.Parameters.Add("version", JObject.FromObject(new YPipelineParameter {
                DefaultValue = version
            }));


            //var jsonPipeline = JsonConvert.SerializeObject(copyPipeline);

            // Get the response. we may want to create a real class for this result ?
            var pipeline = await this.client.ProcessRequestManagementAsync <JObject>(
                pipelinePathUri, pipelineQuery, copyPipeline, HttpMethod.Put).ConfigureAwait(false);


            var triggerName = $"trg_{pipelineName}";

            var triggerPathUri = $"/subscriptions/{options.SubscriptionId}" +
                                 $"/resourceGroups/{engine.ResourceGroupName}/providers/Microsoft.DataFactory" +
                                 $"/factories/{engine.FactoryName}/triggers/{triggerName}";


            var trigger = new YTrigger();

            var pipelineRef = new YTriggerTriggerPipeline();

            pipelineRef.PipelineReference.ReferenceName = pipelineName;
            pipelineRef.Parameters = new JObject {
                { "windowStart", "@trigger().startTime" },
                { "fullLoad", "0" }
            };

            trigger.Properties.Pipelines.Add(pipelineRef);

            trigger.Properties.RuntimeState = "Started";
            trigger.Properties.Type         = "ScheduleTrigger";

            // Get the response. we may want to create a real class for this result ?
            var newTrigger = await this.client.ProcessRequestManagementAsync <JObject>(
                triggerPathUri, pipelineQuery, trigger, HttpMethod.Put).ConfigureAwait(false);


            var triggerStartUri = $"/subscriptions/{options.SubscriptionId}" +
                                  $"/resourceGroups/{engine.ResourceGroupName}/providers/Microsoft.DataFactory" +
                                  $"/factories/{engine.FactoryName}/triggers/{triggerName}/start";

            // Get the response. we may want to create a real class for this result ?
            var newTriggerStarted = await this.client.ProcessRequestManagementAsync <JObject>(
                triggerStartUri, pipelineQuery, null, HttpMethod.Post).ConfigureAwait(false);
        }
Пример #15
0
        private async Task PostMetricAsync(YEngine engine, YMetricPayload payload, string instrumentationKey)
        {
            using TelemetryConfiguration configuration = TelemetryConfiguration.CreateDefault();
            configuration.InstrumentationKey           = instrumentationKey;

            var telemetryClient = new TelemetryClient(configuration);

            if (payload.Dimensions.Count <= 0)
            {
                var id     = new MetricIdentifier(payload.Namespace, payload.Id);
                var metric = telemetryClient.GetMetric(id);
                metric.TrackValue(payload.Value);
            }
            else if (payload.Dimensions.Count == 1)
            {
                var id     = new MetricIdentifier(payload.Namespace, payload.Id, payload.Dimensions[0].Name);
                var metric = telemetryClient.GetMetric(id);
                metric.TrackValue(payload.Value, payload.Dimensions[0].Value);
            }
            else if (payload.Dimensions.Count == 2)
            {
                var id     = new MetricIdentifier(payload.Namespace, payload.Id, payload.Dimensions[0].Name, payload.Dimensions[1].Name);
                var metric = telemetryClient.GetMetric(id);
                metric.TrackValue(payload.Value, payload.Dimensions[0].Value, payload.Dimensions[1].Value);
            }
            else if (payload.Dimensions.Count == 3)
            {
                var id     = new MetricIdentifier(payload.Namespace, payload.Id, payload.Dimensions[0].Name, payload.Dimensions[1].Name, payload.Dimensions[2].Name);
                var metric = telemetryClient.GetMetric(id);
                metric.TrackValue(payload.Value, payload.Dimensions[0].Value, payload.Dimensions[1].Value, payload.Dimensions[2].Value);
            }
            else if (payload.Dimensions.Count == 4)
            {
                var id = new MetricIdentifier(payload.Namespace, payload.Id, payload.Dimensions[0].Name, payload.Dimensions[1].Name, payload.Dimensions[2].Name
                                              , payload.Dimensions[3].Name);
                var metric = telemetryClient.GetMetric(id);
                metric.TrackValue(payload.Value, payload.Dimensions[0].Value, payload.Dimensions[1].Value, payload.Dimensions[2].Value
                                  , payload.Dimensions[3].Value);
            }
            else if (payload.Dimensions.Count == 5)
            {
                var id = new MetricIdentifier(payload.Namespace, payload.Id, payload.Dimensions[0].Name, payload.Dimensions[1].Name, payload.Dimensions[2].Name
                                              , payload.Dimensions[3].Name, payload.Dimensions[4].Name);
                var metric = telemetryClient.GetMetric(id);
                metric.TrackValue(payload.Value, payload.Dimensions[0].Value, payload.Dimensions[1].Value, payload.Dimensions[2].Value
                                  , payload.Dimensions[3].Value, payload.Dimensions[4].Value);
            }
            else if (payload.Dimensions.Count == 6)
            {
                var id = new MetricIdentifier(payload.Namespace, payload.Id, payload.Dimensions[0].Name, payload.Dimensions[1].Name, payload.Dimensions[2].Name
                                              , payload.Dimensions[3].Name, payload.Dimensions[4].Name, payload.Dimensions[5].Name);
                var metric = telemetryClient.GetMetric(id);
                metric.TrackValue(payload.Value, payload.Dimensions[0].Value, payload.Dimensions[1].Value, payload.Dimensions[2].Value
                                  , payload.Dimensions[3].Value, payload.Dimensions[4].Value, payload.Dimensions[5].Value);
            }
            else if (payload.Dimensions.Count == 7)
            {
                var id = new MetricIdentifier(payload.Namespace, payload.Id, payload.Dimensions[0].Name, payload.Dimensions[1].Name, payload.Dimensions[2].Name
                                              , payload.Dimensions[3].Name, payload.Dimensions[4].Name, payload.Dimensions[5].Name, payload.Dimensions[6].Name);
                var metric = telemetryClient.GetMetric(id);
                metric.TrackValue(payload.Value, payload.Dimensions[0].Value, payload.Dimensions[1].Value, payload.Dimensions[2].Value
                                  , payload.Dimensions[3].Value, payload.Dimensions[4].Value, payload.Dimensions[5].Value, payload.Dimensions[6].Value);
            }
            else if (payload.Dimensions.Count == 8)
            {
                var id = new MetricIdentifier(payload.Namespace, payload.Id, payload.Dimensions[0].Name, payload.Dimensions[1].Name, payload.Dimensions[2].Name
                                              , payload.Dimensions[3].Name, payload.Dimensions[4].Name, payload.Dimensions[5].Name, payload.Dimensions[6].Name, payload.Dimensions[7].Name);
                var metric = telemetryClient.GetMetric(id);
                metric.TrackValue(payload.Value, payload.Dimensions[0].Value, payload.Dimensions[1].Value, payload.Dimensions[2].Value
                                  , payload.Dimensions[3].Value, payload.Dimensions[4].Value, payload.Dimensions[5].Value, payload.Dimensions[6].Value, payload.Dimensions[7].Value);
            }
            else if (payload.Dimensions.Count == 9)
            {
                var id = new MetricIdentifier(payload.Namespace, payload.Id, payload.Dimensions[0].Name, payload.Dimensions[1].Name, payload.Dimensions[2].Name
                                              , payload.Dimensions[3].Name, payload.Dimensions[4].Name, payload.Dimensions[5].Name, payload.Dimensions[6].Name, payload.Dimensions[7].Name
                                              , payload.Dimensions[8].Name);
                var metric = telemetryClient.GetMetric(id);
                metric.TrackValue(payload.Value, payload.Dimensions[0].Value, payload.Dimensions[1].Value, payload.Dimensions[2].Value
                                  , payload.Dimensions[3].Value, payload.Dimensions[4].Value, payload.Dimensions[5].Value, payload.Dimensions[6].Value, payload.Dimensions[7].Value
                                  , payload.Dimensions[8].Value);
            }
            else if (payload.Dimensions.Count >= 10)
            {
                var id = new MetricIdentifier(payload.Namespace, payload.Id, payload.Dimensions[0].Name, payload.Dimensions[1].Name, payload.Dimensions[2].Name
                                              , payload.Dimensions[3].Name, payload.Dimensions[4].Name, payload.Dimensions[5].Name, payload.Dimensions[6].Name, payload.Dimensions[7].Name
                                              , payload.Dimensions[8].Name, payload.Dimensions[9].Name);
                var metric = telemetryClient.GetMetric(id);
                metric.TrackValue(payload.Value, payload.Dimensions[0].Value, payload.Dimensions[1].Value, payload.Dimensions[2].Value
                                  , payload.Dimensions[3].Value, payload.Dimensions[4].Value, payload.Dimensions[5].Value, payload.Dimensions[6].Value, payload.Dimensions[7].Value
                                  , payload.Dimensions[8].Value, payload.Dimensions[9].Value);
            }

            telemetryClient.Flush();
        }