public async Task <ActionResult <List <YEntity> > > GetEntitiesFromDaemonAsync(Guid engineId)
{
var userObjectId = this.User.GetObjectId();
if (string.IsNullOrEmpty(userObjectId))
{
return(new UnauthorizedObjectResult("Daemon id unknown"));
}
if (userObjectId != this.options.ClientObjectId)
{
return(new UnauthorizedObjectResult("This web api should be called only from a daemon application using the correct Client Id / Client Secret"));
}
var engine = await this.engineProvider.GetEngineAsync(engineId).ConfigureAwait(false);
if (engine == null)
{
throw new Exception("Engine does not exists");
}
var resourceGroupName = engine.ResourceGroupName;
var factoryName = engine.FactoryName;
var pathUri = $"/subscriptions/{options.SubscriptionId}/resourceGroups/{resourceGroupName}" +
$"/providers/Microsoft.DataFactory/factories/{factoryName}" +
$"/datasets";
var query = $"api-version={DataFactoryApiVersion}";
// Get the response. we may want to create a real class for this result ?
var datasetsTokenResponse = await this.client.ProcessRequestManagementAsync <YEntities>(
pathUri, query).ConfigureAwait(false);
if (datasetsTokenResponse.StatusCode == HttpStatusCode.NotFound)
{
return(new NotFoundResult());
}
var entities = datasetsTokenResponse.Value.Value;
return(entities.Select(e => YEntityFactory.GetTypedEntity(e)).ToList());
}
private async Task CreatePipelineAsync(YEngine engine, YEntity entity)
{
// -------------------------
// PIPELINE
string version = entity.Version.Replace(".", "_");
if (string.IsNullOrEmpty(version))
{
version = "v1";
}
// try to create a Copy Pipeline
string pipelineName = $"{entity.DataSourceName.ToLower()}_{entity.Name.ToLower()}_{version.ToLower()}";
var pipelinePathUri = $"/subscriptions/{options.SubscriptionId}" +
$"/resourceGroups/{engine.ResourceGroupName}/providers/Microsoft.DataFactory" +
$"/factories/{engine.FactoryName}/pipelines/{pipelineName}";
var pipelineQuery = $"api-version={DataFactoryApiVersion}";
var copyPipeline = new YPipeline
{
Name = pipelineName
};
// Copy Pipeline
var copyActivity = new YPipelineActivity
{
Name = "Loading",
Type = "Copy"
};
var source = new YPipelineSource
{
Type = entity.EntityType switch
{
YEntityType.DelimitedText => "DelimitedTextSource",
YEntityType.AzureSqlTable => "AzureSqlSource",
YEntityType.Parquet => "ParquetSource",
_ => "DelimitedTextSource",
}
};
if (entity.EntityType == YEntityType.Parquet || entity.EntityType == YEntityType.DelimitedText)
{
source.StoreSettings = new YPipelineStoreSettings();
source.StoreSettings.Recursive = true;
source.StoreSettings.WildcardFileName = "*";
source.StoreSettings.Type = "AzureBlobStorageReadSettings";
}
else if (entity.EntityType == YEntityType.AzureSqlTable)
{
// only creates if we have the entity supporting it AND request by the method
if (entity.Mode == "Delta")
{
var sqlEntity = YEntityFactory.GetTypedEntity(entity) as YEntityAzureSqlTable;
source.SqlReaderQuery = new YValueType
{
Type = "Expression",
Value = $"Declare @startDate Datetime = '@{{formatDateTime(pipeline().parameters.windowStart, 'yyyy-MM-dd HH:mm')}}'; " +
$"Declare @fullLoad boolean = @{{pipeline().parameters.fullLoad}}; " +
$"Select * From [{sqlEntity.Schema}].[{sqlEntity.Table}] " +
$"Where ([ModifiedDate] >= @startDate And @fullLoad = 0) OR (@fullLoad = 1)"
};
}
source.PartitionOption = "none";
}
else
{
source.PartitionOption = "none";
}
copyActivity.TypeProperties.Add("source", JObject.FromObject(source));
var sink = new YPipelineSink {
Type = "ParquetSink"
};
sink.StoreSettings.Type = "AzureBlobFSWriteSettings";
sink.FormatSettings.Type = "ParquetWriteSettings";
copyActivity.TypeProperties.Add("sink", JObject.FromObject(sink));
copyActivity.TypeProperties.Add("enableStaging", false);
copyActivity.Inputs = new List <YPipelineReference>();
copyActivity.Inputs.Add(new YPipelineReference
{
ReferenceName = entity.Name,
Type = "DatasetReference"
});
var output = new YPipelineOutput
{
ReferenceName = "destinationOutput",
Type = "DatasetReference",
};
output.Parameters.FileSystem.Type = "Expression";
output.Parameters.FileSystem.Value = "@pipeline().parameters.destinationContainer";
output.Parameters.FolderPath.Type = "Expression";
output.Parameters.FolderPath.Value = "@{pipeline().parameters.destinationFolderPath}/@{formatDateTime(pipeline().parameters.windowStart,'yyyy')}/@{formatDateTime(pipeline().parameters.windowStart,'MM')}/@{formatDateTime(pipeline().parameters.windowStart,'dd')}/@{formatDateTime(pipeline().parameters.windowStart,'HH')}";
copyActivity.Outputs = new List <YPipelineOutput>();
copyActivity.Outputs.Add(output);
copyPipeline.Properties.Activities.Add(copyActivity);
// databricks
var dbricksActivity = new YPipelineActivity
{
Name = "Transform",
Type = "DatabricksNotebook"
};
var dependOn = new YPipelineDependsOn {
Activity = "Loading"
};
dependOn.DependencyConditions.Add("Succeeded");
dbricksActivity.DependsOn.Add(dependOn);
dbricksActivity.LinkedServiceName = new YPipelineReference {
ReferenceName = $"dsDatabricks-{engine.ClusterName}", Type = "LinkedServiceReference"
};
dbricksActivity.TypeProperties.Add("notebookPath", "/Shared/main");
dbricksActivity.TypeProperties.Add("baseParameters", new JObject {
{ "entityName", new JObject {
{ "value", "@{pipeline().parameters.entityName}" }, { "type", "Expression" }
} },
{ "inputPath", new JObject {
{ "value", "@{concat(pipeline().parameters.destinationFolderPath, '/', formatDateTime(pipeline().parameters.windowStart,'yyyy'), '/', formatDateTime(pipeline().parameters.windowStart,'MM'), '/', formatDateTime(pipeline().parameters.windowStart,'dd'), '/', formatDateTime(pipeline().parameters.windowStart,'HH'))}" }, { "type", "Expression" }
} },
{ "outputPath", new JObject {
{ "value", "@{pipeline().parameters.deltaFolderPath}" }, { "type", "Expression" }
} },
{ "inputContainer", new JObject {
{ "value", "@{pipeline().parameters.destinationContainer}" }, { "type", "Expression" }
} },
{ "outputContainer", new JObject {
{ "value", "@{pipeline().parameters.deltaContainer}" }, { "type", "Expression" }
} },
{ "dataSourceName", new JObject {
{ "value", "@{pipeline().parameters.dataSourceName}" }, { "type", "Expression" }
} },
{ "version", new JObject {
{ "value", "@{pipeline().parameters.version}" }, { "type", "Expression" }
} }
});
copyPipeline.Properties.Activities.Add(dbricksActivity);
copyPipeline.Properties.Parameters.Add("windowStart", JObject.FromObject(new YPipelineParameter {
DefaultValue = DateTime.Now
}));
copyPipeline.Properties.Parameters.Add("fullLoad", JObject.FromObject(new YPipelineParameter {
DefaultValue = "1"
}));
copyPipeline.Properties.Parameters.Add("destinationContainer", JObject.FromObject(new YPipelineParameter {
DefaultValue = "bronze"
}));
copyPipeline.Properties.Parameters.Add("destinationFolderPath", JObject.FromObject(new YPipelineParameter {
DefaultValue = $"{entity.DataSourceName}/{entity.Name}/{version}"
}));
copyPipeline.Properties.Parameters.Add("deltaContainer", JObject.FromObject(new YPipelineParameter {
DefaultValue = "silver"
}));
copyPipeline.Properties.Parameters.Add("deltaFolderPath", JObject.FromObject(new YPipelineParameter {
DefaultValue = $"{entity.DataSourceName}/{entity.Name}/{version}"
}));
copyPipeline.Properties.Parameters.Add("engineId", JObject.FromObject(new YPipelineParameter {
DefaultValue = engine.Id
}));
copyPipeline.Properties.Parameters.Add("entityName", JObject.FromObject(new YPipelineParameter {
DefaultValue = entity.Name
}));
copyPipeline.Properties.Parameters.Add("dataSourceName", JObject.FromObject(new YPipelineParameter {
DefaultValue = entity.DataSourceName
}));
copyPipeline.Properties.Parameters.Add("version", JObject.FromObject(new YPipelineParameter {
DefaultValue = version
}));
//var jsonPipeline = JsonConvert.SerializeObject(copyPipeline);
// Get the response. we may want to create a real class for this result ?
var pipeline = await this.client.ProcessRequestManagementAsync <JObject>(
pipelinePathUri, pipelineQuery, copyPipeline, HttpMethod.Put).ConfigureAwait(false);
var triggerName = $"trg_{pipelineName}";
var triggerPathUri = $"/subscriptions/{options.SubscriptionId}" +
$"/resourceGroups/{engine.ResourceGroupName}/providers/Microsoft.DataFactory" +
$"/factories/{engine.FactoryName}/triggers/{triggerName}";
var trigger = new YTrigger();
var pipelineRef = new YTriggerTriggerPipeline();
pipelineRef.PipelineReference.ReferenceName = pipelineName;
pipelineRef.Parameters = new JObject {
{ "windowStart", "@trigger().startTime" },
{ "fullLoad", "0" }
};
trigger.Properties.Pipelines.Add(pipelineRef);
trigger.Properties.RuntimeState = "Started";
trigger.Properties.Type = "ScheduleTrigger";
// Get the response. we may want to create a real class for this result ?
var newTrigger = await this.client.ProcessRequestManagementAsync <JObject>(
triggerPathUri, pipelineQuery, trigger, HttpMethod.Put).ConfigureAwait(false);
var triggerStartUri = $"/subscriptions/{options.SubscriptionId}" +
$"/resourceGroups/{engine.ResourceGroupName}/providers/Microsoft.DataFactory" +
$"/factories/{engine.FactoryName}/triggers/{triggerName}/start";
// Get the response. we may want to create a real class for this result ?
var newTriggerStarted = await this.client.ProcessRequestManagementAsync <JObject>(
triggerStartUri, pipelineQuery, null, HttpMethod.Post).ConfigureAwait(false);
}