Ejemplo n.º 1
0
        public Task <YJsonResult <YEntityUnknown> > AddEntityAsync(Guid engineId, string dataSourceName, string entityName, [FromBody] YEntity payload)
        {
            return(YExecuteAsync(async() =>
            {
                var response = await this.client.ProcessRequestApiAsync <YEntityUnknown>(
                    $"api/DataFactories/{engineId}/links/{dataSourceName}/entities/{entityName}",
                    null, payload, HttpMethod.Put).ConfigureAwait(false);

                var entity = response.Value;

                return entity;
            }));
        }
Ejemplo n.º 2
0
        private async Task CreatePipelineAsync(YEngine engine, YEntity entity)
        {
            // -------------------------
            // PIPELINE


            string version = entity.Version.Replace(".", "_");


            if (string.IsNullOrEmpty(version))
            {
                version = "v1";
            }

            // try to create a Copy Pipeline
            string pipelineName = $"{entity.DataSourceName.ToLower()}_{entity.Name.ToLower()}_{version.ToLower()}";


            var pipelinePathUri = $"/subscriptions/{options.SubscriptionId}" +
                                  $"/resourceGroups/{engine.ResourceGroupName}/providers/Microsoft.DataFactory" +
                                  $"/factories/{engine.FactoryName}/pipelines/{pipelineName}";

            var pipelineQuery = $"api-version={DataFactoryApiVersion}";


            var copyPipeline = new YPipeline
            {
                Name = pipelineName
            };

            // Copy Pipeline

            var copyActivity = new YPipelineActivity
            {
                Name = "Loading",
                Type = "Copy"
            };

            var source = new YPipelineSource
            {
                Type = entity.EntityType switch
                {
                    YEntityType.DelimitedText => "DelimitedTextSource",
                    YEntityType.AzureSqlTable => "AzureSqlSource",
                    YEntityType.Parquet => "ParquetSource",
                    _ => "DelimitedTextSource",
                }
            };

            if (entity.EntityType == YEntityType.Parquet || entity.EntityType == YEntityType.DelimitedText)
            {
                source.StoreSettings                  = new YPipelineStoreSettings();
                source.StoreSettings.Recursive        = true;
                source.StoreSettings.WildcardFileName = "*";
                source.StoreSettings.Type             = "AzureBlobStorageReadSettings";
            }
            else if (entity.EntityType == YEntityType.AzureSqlTable)
            {
                // only creates if we have the entity supporting it AND request by the method
                if (entity.Mode == "Delta")
                {
                    var sqlEntity = YEntityFactory.GetTypedEntity(entity) as YEntityAzureSqlTable;

                    source.SqlReaderQuery = new YValueType
                    {
                        Type  = "Expression",
                        Value = $"Declare @startDate Datetime = '@{{formatDateTime(pipeline().parameters.windowStart, 'yyyy-MM-dd HH:mm')}}'; " +
                                $"Declare @fullLoad boolean = @{{pipeline().parameters.fullLoad}}; " +
                                $"Select * From [{sqlEntity.Schema}].[{sqlEntity.Table}] " +
                                $"Where ([ModifiedDate] >= @startDate And @fullLoad = 0) OR (@fullLoad = 1)"
                    };
                }
                source.PartitionOption = "none";
            }
            else
            {
                source.PartitionOption = "none";
            }

            copyActivity.TypeProperties.Add("source", JObject.FromObject(source));

            var sink = new YPipelineSink {
                Type = "ParquetSink"
            };

            sink.StoreSettings.Type  = "AzureBlobFSWriteSettings";
            sink.FormatSettings.Type = "ParquetWriteSettings";
            copyActivity.TypeProperties.Add("sink", JObject.FromObject(sink));

            copyActivity.TypeProperties.Add("enableStaging", false);


            copyActivity.Inputs = new List <YPipelineReference>();
            copyActivity.Inputs.Add(new YPipelineReference
            {
                ReferenceName = entity.Name,
                Type          = "DatasetReference"
            });

            var output = new YPipelineOutput
            {
                ReferenceName = "destinationOutput",
                Type          = "DatasetReference",
            };

            output.Parameters.FileSystem.Type  = "Expression";
            output.Parameters.FileSystem.Value = "@pipeline().parameters.destinationContainer";
            output.Parameters.FolderPath.Type  = "Expression";
            output.Parameters.FolderPath.Value = "@{pipeline().parameters.destinationFolderPath}/@{formatDateTime(pipeline().parameters.windowStart,'yyyy')}/@{formatDateTime(pipeline().parameters.windowStart,'MM')}/@{formatDateTime(pipeline().parameters.windowStart,'dd')}/@{formatDateTime(pipeline().parameters.windowStart,'HH')}";

            copyActivity.Outputs = new List <YPipelineOutput>();
            copyActivity.Outputs.Add(output);

            copyPipeline.Properties.Activities.Add(copyActivity);

            // databricks

            var dbricksActivity = new YPipelineActivity
            {
                Name = "Transform",
                Type = "DatabricksNotebook"
            };

            var dependOn = new YPipelineDependsOn {
                Activity = "Loading"
            };

            dependOn.DependencyConditions.Add("Succeeded");
            dbricksActivity.DependsOn.Add(dependOn);

            dbricksActivity.LinkedServiceName = new YPipelineReference {
                ReferenceName = $"dsDatabricks-{engine.ClusterName}", Type = "LinkedServiceReference"
            };

            dbricksActivity.TypeProperties.Add("notebookPath", "/Shared/main");
            dbricksActivity.TypeProperties.Add("baseParameters", new JObject {
                { "entityName", new JObject {
                      { "value", "@{pipeline().parameters.entityName}" }, { "type", "Expression" }
                  } },
                { "inputPath", new JObject {
                      { "value", "@{concat(pipeline().parameters.destinationFolderPath, '/', formatDateTime(pipeline().parameters.windowStart,'yyyy'), '/', formatDateTime(pipeline().parameters.windowStart,'MM'), '/', formatDateTime(pipeline().parameters.windowStart,'dd'), '/', formatDateTime(pipeline().parameters.windowStart,'HH'))}" }, { "type", "Expression" }
                  } },
                { "outputPath", new JObject {
                      { "value", "@{pipeline().parameters.deltaFolderPath}" }, { "type", "Expression" }
                  } },
                { "inputContainer", new JObject {
                      { "value", "@{pipeline().parameters.destinationContainer}" }, { "type", "Expression" }
                  } },
                { "outputContainer", new JObject {
                      { "value", "@{pipeline().parameters.deltaContainer}" }, { "type", "Expression" }
                  } },
                { "dataSourceName", new JObject {
                      { "value", "@{pipeline().parameters.dataSourceName}" }, { "type", "Expression" }
                  } },
                { "version", new JObject {
                      { "value", "@{pipeline().parameters.version}" }, { "type", "Expression" }
                  } }
            });

            copyPipeline.Properties.Activities.Add(dbricksActivity);


            copyPipeline.Properties.Parameters.Add("windowStart", JObject.FromObject(new YPipelineParameter {
                DefaultValue = DateTime.Now
            }));
            copyPipeline.Properties.Parameters.Add("fullLoad", JObject.FromObject(new YPipelineParameter {
                DefaultValue = "1"
            }));
            copyPipeline.Properties.Parameters.Add("destinationContainer", JObject.FromObject(new YPipelineParameter {
                DefaultValue = "bronze"
            }));
            copyPipeline.Properties.Parameters.Add("destinationFolderPath", JObject.FromObject(new YPipelineParameter {
                DefaultValue = $"{entity.DataSourceName}/{entity.Name}/{version}"
            }));
            copyPipeline.Properties.Parameters.Add("deltaContainer", JObject.FromObject(new YPipelineParameter {
                DefaultValue = "silver"
            }));
            copyPipeline.Properties.Parameters.Add("deltaFolderPath", JObject.FromObject(new YPipelineParameter {
                DefaultValue = $"{entity.DataSourceName}/{entity.Name}/{version}"
            }));
            copyPipeline.Properties.Parameters.Add("engineId", JObject.FromObject(new YPipelineParameter {
                DefaultValue = engine.Id
            }));
            copyPipeline.Properties.Parameters.Add("entityName", JObject.FromObject(new YPipelineParameter {
                DefaultValue = entity.Name
            }));
            copyPipeline.Properties.Parameters.Add("dataSourceName", JObject.FromObject(new YPipelineParameter {
                DefaultValue = entity.DataSourceName
            }));
            copyPipeline.Properties.Parameters.Add("version", JObject.FromObject(new YPipelineParameter {
                DefaultValue = version
            }));


            //var jsonPipeline = JsonConvert.SerializeObject(copyPipeline);

            // Get the response. we may want to create a real class for this result ?
            var pipeline = await this.client.ProcessRequestManagementAsync <JObject>(
                pipelinePathUri, pipelineQuery, copyPipeline, HttpMethod.Put).ConfigureAwait(false);


            var triggerName = $"trg_{pipelineName}";

            var triggerPathUri = $"/subscriptions/{options.SubscriptionId}" +
                                 $"/resourceGroups/{engine.ResourceGroupName}/providers/Microsoft.DataFactory" +
                                 $"/factories/{engine.FactoryName}/triggers/{triggerName}";


            var trigger = new YTrigger();

            var pipelineRef = new YTriggerTriggerPipeline();

            pipelineRef.PipelineReference.ReferenceName = pipelineName;
            pipelineRef.Parameters = new JObject {
                { "windowStart", "@trigger().startTime" },
                { "fullLoad", "0" }
            };

            trigger.Properties.Pipelines.Add(pipelineRef);

            trigger.Properties.RuntimeState = "Started";
            trigger.Properties.Type         = "ScheduleTrigger";

            // Get the response. we may want to create a real class for this result ?
            var newTrigger = await this.client.ProcessRequestManagementAsync <JObject>(
                triggerPathUri, pipelineQuery, trigger, HttpMethod.Put).ConfigureAwait(false);


            var triggerStartUri = $"/subscriptions/{options.SubscriptionId}" +
                                  $"/resourceGroups/{engine.ResourceGroupName}/providers/Microsoft.DataFactory" +
                                  $"/factories/{engine.FactoryName}/triggers/{triggerName}/start";

            // Get the response. we may want to create a real class for this result ?
            var newTriggerStarted = await this.client.ProcessRequestManagementAsync <JObject>(
                triggerStartUri, pipelineQuery, null, HttpMethod.Post).ConfigureAwait(false);
        }