public static IActionResult ExtractPipelineLog(
[HttpTrigger(AuthorizationLevel.Anonymous, "get", Route = null)] HttpRequest req,
ILogger log)
{
log.LogInformation("Extracting pipeline log.");
string pipelineName = req.Query["pipelineName"];
if (pipelineName == null)
{
return(new BadRequestObjectResult("Please pass a pipelineName on the query string"));
}
var client = buildClient();
RunFilterParameters filterParameters = new RunFilterParameters();
filterParameters.LastUpdatedAfter = DateTime.MinValue;
filterParameters.LastUpdatedBefore = DateTime.MaxValue;
PipelineRunsQueryResponse pipelineRuns = client.PipelineRuns.QueryByFactory(
getResourceGroup(), getDataFactoryName(), filterParameters);
PipelineRun latestPipelineRun = null;
foreach (var run in pipelineRuns.Value)
{
if (run.PipelineName == pipelineName &&
((latestPipelineRun == null) || (run.IsLatest == true && run.LastUpdated > latestPipelineRun.LastUpdated)))
{
latestPipelineRun = run;
}
}
return((ActionResult) new OkObjectResult(JsonConvert.SerializeObject(latestPipelineRun)));
}
public void PipelineRuns_Get()
{
RunTest("PipelineRuns_Get", (example, client, responseCode) =>
{
PipelineRun response = client.PipelineRuns.Get(RGN(example), FN(example), GetTypedParameter <string>(example, client, "runId"));
CheckResponseBody(example, client, responseCode, response);
});
}
public PSPipelineRun(PipelineRun pipelineRun, string resourceGroupName, string factoryName)
{
if (pipelineRun == null)
{
throw new ArgumentNullException("pipelineRun");
}
this.pipelineRun = pipelineRun;
this.ResourceGroupName = resourceGroupName;
this.DataFactoryName = factoryName;
}
public static void initialise()
{
DataFactoryManagementClient client = ADF.GetAuthUser();
ADF.CreateADF(client);
ADF.CreateBlobLS(client);
ADF.CreateDS(client);
ADF.CreatePipeline(client);
CreateRunResponse runResponse = ADF.CreatePipelineRun(client);
PipelineRun pipelineRun = ADF.MonitorPipelineStatus(client, runResponse);
ADF.MonitorActivityStatus(client, runResponse, pipelineRun);
}
public async Task TestGet()
{
PipelineClient pipelineClient = CreatePipelineClient();
PipelineRunClient runClient = CreatePipelineRunClient();
await using DisposablePipeline pipeline = await DisposablePipeline.Create(pipelineClient, this.Recording);
CreateRunResponse runResponse = await pipelineClient.CreatePipelineRunAsync(pipeline.Name);
Assert.NotNull(runResponse.RunId);
PipelineRun run = await runClient.GetPipelineRunAsync(runResponse.RunId);
Assert.AreEqual(run.RunId, runResponse.RunId);
Assert.NotNull(run.Status);
}
public PSPipelineRun(PipelineRun pipelineRun, string workspaceName)
{
this.WorkspaceName = workspaceName;
this.Message = pipelineRun?.Message;
this.Status = pipelineRun?.Status;
this.DurationInMs = pipelineRun?.DurationInMs;
this.RunEnd = pipelineRun?.RunEnd;
this.RunStart = pipelineRun?.RunStart;
this.LastUpdated = pipelineRun?.LastUpdated;
this.InvokedBy = new PSPipelineRunInvokedBy(pipelineRun?.InvokedBy);
this.Parameters = pipelineRun?.Parameters;
this.PipelineName = pipelineRun?.PipelineName;
this.IsLatest = pipelineRun?.IsLatest;
this.RunGroupId = pipelineRun?.RunGroupId;
this.RunId = pipelineRun?.RunId;
this.AdditionalProperties = pipelineRun?.AdditionalProperties;
}
private void CapturePipelineRuns_Get(string runId)
{
interceptor.CurrentExampleName = "PipelineRuns_Get";
PipelineRun run = client.PipelineRuns.Get(secrets.ResourceGroupName, secrets.FactoryName, runId);
}
static void Main(string[] args)
{
//Authentification auprès d'Azure avec l'application svc_adf
AuthenticationContext context = new AuthenticationContext("https://login.windows.net/" + _repertoireId);
ClientCredential cc = new ClientCredential(_applicationId, _applicationKey);
AuthenticationResult result = context.AcquireTokenAsync("https://management.azure.com/", cc).Result;
ServiceClientCredentials cred = new TokenCredentials(result.AccessToken);
DataFactoryManagementClient ADFclient = new DataFactoryManagementClient(cred)
{
SubscriptionId = _abonnementId
};
//Création d'une Azure Data Factory
Factory dataFactory = new Factory
{
Location = _region,
Identity = new FactoryIdentity()
};
ADFclient.Factories.CreateOrUpdate(_ressourceGroup, _dataFactoryName, dataFactory);
Console.WriteLine(SafeJsonConvert.SerializeObject(dataFactory, ADFclient.SerializationSettings));
var toto = ADFclient.Factories.Get(_ressourceGroup, _dataFactoryName).ProvisioningState;
while (ADFclient.Factories.Get(_ressourceGroup, _dataFactoryName).ProvisioningState == "PendingCreation")
{
System.Threading.Thread.Sleep(1000);
Console.WriteLine("*");
}
////Création d'un Integration Runtime Auto-Hébergé
//IntegrationRuntimeResource integrationRuntimeResource = new IntegrationRuntimeResource(
//new SelfHostedIntegrationRuntime
//{
// Description = "L'Integration Runtime du projet ..."
//}
//);
//ADFclient.IntegrationRuntimes.CreateOrUpdate(_ressourceGroup, _dataFactoryName, _IntegrationRuntimeName, integrationRuntimeResource);
//Console.WriteLine(SafeJsonConvert.SerializeObject(integrationRuntimeResource, ADFclient.SerializationSettings));
//Console.WriteLine("Authkey : " + ADFclient.IntegrationRuntimes.ListAuthKeys(_ressourceGroup, _dataFactoryName, _IntegrationRuntimeName).AuthKey1);
//Création service lié File System on premise
IntegrationRuntimeReference integrationRuntimeReference = new IntegrationRuntimeReference(_IntegrationRuntimeName);
SecureString secureString = new SecureString("MonPassword");
LinkedServiceResource FS_PartageOnPremise = new LinkedServiceResource(
new FileServerLinkedService
{
Description = "Service lié référençant un espace partagé dans le réseau privé de l'entreprise",
ConnectVia = integrationRuntimeReference,
Host = @"\\IRAutoHeberge\Dépôt",
UserId = "chsauget",
Password = secureString
}
);
ADFclient.LinkedServices.CreateOrUpdate(_ressourceGroup, _dataFactoryName, _FS_PartageOnPremiseName, FS_PartageOnPremise);
Console.WriteLine(SafeJsonConvert.SerializeObject(FS_PartageOnPremise, ADFclient.SerializationSettings));
//Création service lié Azure SQLDB
SecureString SQLsecureString = new SecureString("integrated security=False;encrypt=True;connection timeout=30;data source=adflivre.database.windows.net;initial catalog=advwrks;user id=chsauget;Password=toto");
LinkedServiceResource SQDB_AdventureWorks = new LinkedServiceResource(
new AzureSqlDatabaseLinkedService
{
Description = "Service lié référençant un espace partagé dans le réseau privé de l'entreprise",
ConnectionString = SQLsecureString,
}
);
ADFclient.LinkedServices.CreateOrUpdate(_ressourceGroup, _dataFactoryName, _SQDB_AdventureWorksName, SQDB_AdventureWorks);
Console.WriteLine(SafeJsonConvert.SerializeObject(SQDB_AdventureWorks, ADFclient.SerializationSettings));
//Création jeu de données FS_Customer
DatasetResource FS_Customer = new DatasetResource(
new FileShareDataset
{
LinkedServiceName = new LinkedServiceReference
{
ReferenceName = _FS_PartageOnPremiseName
}
, FolderPath = "AdventureWorks CSV"
, FileName = "Customer.csv"
, Format = new TextFormat
{
ColumnDelimiter = "\t",
RowDelimiter = "\n",
FirstRowAsHeader = false
}
, Structure = new List <DatasetDataElement>
{
new DatasetDataElement
{
Name = "CustomerID",
Type = "Int32"
},
new DatasetDataElement
{
Name = "PersonID",
Type = "Int32"
},
new DatasetDataElement
{
Name = "StoreID",
Type = "Int32"
},
new DatasetDataElement
{
Name = "TerritoryID",
Type = "Int32"
},
new DatasetDataElement
{
Name = "AccountNumber",
Type = "String"
},
new DatasetDataElement
{
Name = "rowguid",
Type = "String"
},
new DatasetDataElement
{
Name = "ModifiedDate",
Type = "DateTime"
}
}
}
, name: _FS_CustomerName
);
ADFclient.Datasets.CreateOrUpdate(_ressourceGroup, _dataFactoryName, _FS_CustomerName, FS_Customer);
Console.WriteLine(SafeJsonConvert.SerializeObject(FS_Customer, ADFclient.SerializationSettings));
//Création jeu de données SQDB_Col_Customer
DatasetResource SQDB_Col_Customer = new DatasetResource(
new AzureSqlTableDataset
{
LinkedServiceName = new LinkedServiceReference
{
ReferenceName = _SQDB_AdventureWorksName
},
TableName = "col.Customer"
}
, name: _SQDB_Col_CustomerName
);
ADFclient.Datasets.CreateOrUpdate(_ressourceGroup, _dataFactoryName, _SQDB_AdventureWorksName, SQDB_Col_Customer);
Console.WriteLine(SafeJsonConvert.SerializeObject(SQDB_Col_Customer, ADFclient.SerializationSettings));
//Création de l'activité de copie du fichier Customer
CopyActivity CustomerCopy = new CopyActivity
{
Name = "Copy - Customer"
,
Inputs = new List <DatasetReference> {
new DatasetReference()
{
ReferenceName = _FS_CustomerName
}
}
,
Outputs = new List <DatasetReference> {
new DatasetReference()
{
ReferenceName = _SQDB_Col_CustomerName
}
}
,
Source = new FileSystemSource
{
}
,
Sink = new AzureTableSink
{
}
};
//Création de l'activité de copie du fichier Customer
PipelineResource PipelineCustomer = new PipelineResource
{
Activities = new List <Activity> {
CustomerCopy
}
, Folder = new PipelineFolder {
Name = "AdventureWorks"
}
};
ADFclient.Pipelines.CreateOrUpdate(_ressourceGroup, _dataFactoryName, "Col_Customer", PipelineCustomer);
Console.WriteLine(SafeJsonConvert.SerializeObject(PipelineCustomer, ADFclient.SerializationSettings));
//Demander une execution du pipeline
CreateRunResponse runResponse = ADFclient.Pipelines.CreateRunWithHttpMessagesAsync(_ressourceGroup, _dataFactoryName, "Col_Customer").Result.Body;
//Contrôler l'execution du pipeline
PipelineRun run = ADFclient.PipelineRuns.Get(_ressourceGroup, _dataFactoryName, runResponse.RunId);
while (run.Status == "InProgress")
{
run = ADFclient.PipelineRuns.Get(_ressourceGroup, _dataFactoryName, runResponse.RunId);
Console.WriteLine("Status: " + run.Status);
}
//Déclencheur Quotidien
TriggerResource scheduleTrigger = new TriggerResource(
new ScheduleTrigger
{
Pipelines = new List <TriggerPipelineReference> {
new TriggerPipelineReference {
PipelineReference = new PipelineReference("Col_Customer")
}
}
,
Recurrence = new ScheduleTriggerRecurrence
{
StartTime = DateTime.Parse("2019-03-30T01:00:00Z")
,
Frequency = "Day"
,
Interval = 1
}
}
, name: "Daily_01h_Schedule"
);
ADFclient.Triggers.CreateOrUpdate(_ressourceGroup, _dataFactoryName, "Daily_01h_Schedule", scheduleTrigger);
Console.WriteLine(SafeJsonConvert.SerializeObject(scheduleTrigger, ADFclient.SerializationSettings));
ADFclient.Triggers.BeginStart(_ressourceGroup, _dataFactoryName, "Daily_01h_Schedule");
}
public PSPipelineRun()
{
this.pipelineRun = new PipelineRun();
}
public static void MonitorActivityStatus(DataFactoryManagementClient client, CreateRunResponse runResponse, PipelineRun pipelineRun)
{
Console.WriteLine("Checking copy activity run details...");
RunFilterParameters filterParams = new RunFilterParameters(
DateTime.UtcNow.AddMinutes(-10), DateTime.UtcNow.AddMinutes(10));
ActivityRunsQueryResponse queryResponse = client.ActivityRuns.QueryByPipelineRun(
Constants.resourceGroup, Constants.dataFactoryName, runResponse.RunId, filterParams);
if (pipelineRun.Status == "Succeeded")
{
Console.WriteLine(queryResponse.Value.First().Output);
}
else
{
Console.WriteLine(queryResponse.Value.First().Error);
}
Console.WriteLine("\nPress any key to exit...");
Console.ReadKey();
}
public virtual PSPipelineRun GetPipelineRun(string resourceGroup, string datafactoryName, string pipelineRunId)
{
PipelineRun pipelineRun = this.DataFactoryManagementClient.PipelineRuns.Get(resourceGroup, datafactoryName, pipelineRunId);
return(new PSPipelineRun(pipelineRun, resourceGroup, datafactoryName));
}
public override PipelineErrorDetail GetPipelineRunActivityErrors(PipelineRunRequest request)
{
PipelineRun pipelineRun = _adfManagementClient.PipelineRuns.Get
(
request.ResourceGroupName,
request.OrchestratorName,
request.RunId
);
//Defensive check
PipelineNameCheck(request.PipelineName, pipelineRun.PipelineName);
_logger.LogInformation("Create pipeline Activity Runs query filters.");
RunFilterParameters filterParams = new RunFilterParameters
(
request.ActivityQueryStart,
request.ActivityQueryEnd
);
_logger.LogInformation("Querying ADF pipeline for Activity Runs.");
ActivityRunsQueryResponse queryResponse = _adfManagementClient.ActivityRuns.QueryByPipelineRun
(
request.ResourceGroupName,
request.OrchestratorName,
request.RunId,
filterParams
);
//Create initial output content
PipelineErrorDetail output = new PipelineErrorDetail()
{
PipelineName = request.PipelineName,
ActualStatus = pipelineRun.Status,
RunId = request.RunId,
ResponseCount = queryResponse.Value.Count
};
_logger.LogInformation("Pipeline status: " + pipelineRun.Status);
_logger.LogInformation("Activities found in pipeline response: " + queryResponse.Value.Count.ToString());
//Loop over activities in pipeline run
foreach (ActivityRun activity in queryResponse.Value)
{
if (activity.Error == null)
{
continue; //only want errors
}
//Parse error output to customise output
dynamic outputBlockInner = JsonConvert.DeserializeObject(activity.Error.ToString());
string errorCode = outputBlockInner?.errorCode;
string errorType = outputBlockInner?.failureType;
string errorMessage = outputBlockInner?.message;
_logger.LogInformation("Activity run id: " + activity.ActivityRunId);
_logger.LogInformation("Activity name: " + activity.ActivityName);
_logger.LogInformation("Activity type: " + activity.ActivityType);
_logger.LogInformation("Error message: " + errorMessage);
output.Errors.Add(new FailedActivity()
{
ActivityRunId = activity.ActivityRunId,
ActivityName = activity.ActivityName,
ActivityType = activity.ActivityType,
ErrorCode = errorCode,
ErrorType = errorType,
ErrorMessage = errorMessage
});
}
return(output);
}
public override PipelineErrorDetail GetPipelineRunActivityErrors(PipelineRunRequest request)
{
PipelineRun pipelineRun = _pipelineRunClient.GetPipelineRun
(
request.RunId
);
//Defensive check
PipelineNameCheck(request.PipelineName, pipelineRun.PipelineName);
_logger.LogInformation("Create pipeline Activity Runs query filters.");
RunFilterParameters filterParams = new RunFilterParameters
(
request.ActivityQueryStart,
request.ActivityQueryEnd
);
_logger.LogInformation("Querying SYN pipeline for Activity Runs.");
ActivityRunsQueryResponse queryResponse = _pipelineRunClient.QueryActivityRuns
(
request.PipelineName,
request.RunId,
filterParams
);
//Create initial output content
PipelineErrorDetail output = new PipelineErrorDetail()
{
PipelineName = request.PipelineName,
ActualStatus = pipelineRun.Status,
RunId = request.RunId,
ResponseCount = queryResponse.Value.Count
};
_logger.LogInformation("Pipeline status: " + pipelineRun.Status);
_logger.LogInformation("Activities found in pipeline response: " + queryResponse.Value.Count.ToString());
//Loop over activities in pipeline run
foreach (ActivityRun activity in queryResponse.Value)
{
if (activity.Error == null)
{
continue; //only want errors
}
//Parse error output to customise output
var json = JsonConvert.SerializeObject(activity.Error);
Dictionary <string, object> errorContent = JsonConvert.DeserializeObject <Dictionary <string, object> >(json);
_logger.LogInformation("Activity run id: " + activity.ActivityRunId);
_logger.LogInformation("Activity name: " + activity.ActivityName);
_logger.LogInformation("Activity type: " + activity.ActivityType);
_logger.LogInformation("Error message: " + errorContent["message"].ToString());
output.Errors.Add(new FailedActivity()
{
ActivityRunId = activity.ActivityRunId,
ActivityName = activity.ActivityName,
ActivityType = activity.ActivityType,
ErrorCode = errorContent["errorCode"].ToString(),
ErrorType = errorContent["failureType"].ToString(),
ErrorMessage = errorContent["message"].ToString()
});
}
return(output);
}
