public void ActivityRuns_QueryByPipelineRun()
{
RunTest("ActivityRuns_QueryByPipelineRun", (example, client, responseCode) =>
{
RunFilterParameters filterParams = GetTypedParameter <RunFilterParameters>(example, client, "filterParameters");
ActivityRunsQueryResponse response = client.ActivityRuns.QueryByPipelineRun(RGN(example), FN(example), GetTypedParameter <string>(example, client, "runId"), filterParams);
CheckResponseBody(example, client, responseCode, response);
});
}
public static void MonitorActivityStatus(DataFactoryManagementClient client, CreateRunResponse runResponse, PipelineRun pipelineRun)
{
Console.WriteLine("Checking copy activity run details...");
RunFilterParameters filterParams = new RunFilterParameters(
DateTime.UtcNow.AddMinutes(-10), DateTime.UtcNow.AddMinutes(10));
ActivityRunsQueryResponse queryResponse = client.ActivityRuns.QueryByPipelineRun(
Constants.resourceGroup, Constants.dataFactoryName, runResponse.RunId, filterParams);
if (pipelineRun.Status == "Succeeded")
{
Console.WriteLine(queryResponse.Value.First().Output);
}
else
{
Console.WriteLine(queryResponse.Value.First().Error);
}
Console.WriteLine("\nPress any key to exit...");
Console.ReadKey();
}
static void printPipelines(DataFactoryManagementClient client, PipelineRunsQueryResponse pipelineRuns)
{
var enumerator = pipelineRuns.Value.GetEnumerator();
string runId;
PipelineRun pipelineRun;
while (enumerator.MoveNext())
{
pipelineRun = enumerator.Current;
if (pipelineRun.PipelineName == pipelineName)
{
Console.BackgroundColor = ConsoleColor.White;
Console.ForegroundColor = ConsoleColor.Black;
runId = pipelineRun.RunId;
Console.Write("Start:{1}, End:{2}, Duration(ms):{3}, RunID : {0} - ", runId.ToString(), pipelineRun.RunStart.ToString(), pipelineRun.RunEnd.ToString(), pipelineRun.DurationInMs);
if (pipelineRun.Status != "Succeeded")
{
Console.BackgroundColor = ConsoleColor.Yellow;
Console.ForegroundColor = ConsoleColor.Red;
}
Console.WriteLine("{0}", pipelineRun.Status);
Console.BackgroundColor = ConsoleColor.Black;
Console.ForegroundColor = ConsoleColor.White;
Console.WriteLine("Folder Name :{0}", pipelineRun.Parameters.Count == 0? "N/A" : pipelineRun.Parameters["sourcefolder"].ToString());
Console.WriteLine("File Name :{0}", pipelineRun.Parameters.Count == 0 ? "N/A" : pipelineRun.Parameters["sourcefile"].ToString());
ActivityRunsQueryResponse activities = client.ActivityRuns.QueryByPipelineRun(resourceGroupName, dataFactoryName, runId, new RunFilterParameters(startAfter, endAfter));
printActivities(client, activities);
while (activities.ContinuationToken != null)
{
activities = client.ActivityRuns.QueryByPipelineRun(resourceGroupName, dataFactoryName, runId, new RunFilterParameters(startAfter, endAfter, activities.ContinuationToken));
printActivities(client, activities);
}
}
}
}
static void Main(string[] args)
{
// Set variables
string tenantID = "<your tenant ID>";
string applicationId = "<your application ID>";
string authenticationKey = "<your authentication key for the application>";
string subscriptionId = "<your subscription ID where the data factory resides>";
string resourceGroup = "<your resource group where the data factory resides>";
string region = "<the location of your resource group>";
string dataFactoryName =
"<specify the name of data factory to create. It must be globally unique.>";
string storageAccount = "<your storage account name to copy data>";
string storageKey = "<your storage account key>";
// specify the container and input folder from which all files
// need to be copied to the output folder.
string inputBlobPath =
"<path to existing blob(s) to copy data from, e.g. containername/inputdir>";
//specify the contains and output folder where the files are copied
string outputBlobPath =
"<the blob path to copy data to, e.g. containername/outputdir>";
// name of the Azure Storage linked service, blob dataset, and the pipeline
string storageLinkedServiceName = "AzureStorageLinkedService";
string blobDatasetName = "BlobDataset";
string pipelineName = "Adfv2QuickStartPipeline";
// Authenticate and create a data factory management client
var context = new AuthenticationContext("https://login.microsoftonline.com/" + tenantID);
ClientCredential cc = new ClientCredential(applicationId, authenticationKey);
AuthenticationResult result = context.AcquireTokenAsync(
"https://management.azure.com/", cc).Result;
ServiceClientCredentials cred = new TokenCredentials(result.AccessToken);
var client = new DataFactoryManagementClient(cred)
{
SubscriptionId = subscriptionId
};
// Create a data factory
Console.WriteLine("Creating data factory " + dataFactoryName + "...");
Factory dataFactory = new Factory
{
Location = region,
Identity = new FactoryIdentity()
};
client.Factories.CreateOrUpdate(resourceGroup, dataFactoryName, dataFactory);
Console.WriteLine(
SafeJsonConvert.SerializeObject(dataFactory, client.SerializationSettings));
while (client.Factories.Get(resourceGroup, dataFactoryName).ProvisioningState ==
"PendingCreation")
{
System.Threading.Thread.Sleep(1000);
}
// Create an Azure Storage linked service
Console.WriteLine("Creating linked service " + storageLinkedServiceName + "...");
LinkedServiceResource storageLinkedService = new LinkedServiceResource(
new AzureStorageLinkedService
{
ConnectionString = new SecureString(
"DefaultEndpointsProtocol=https;AccountName=" + storageAccount +
";AccountKey=" + storageKey)
}
);
client.LinkedServices.CreateOrUpdate(
resourceGroup, dataFactoryName, storageLinkedServiceName, storageLinkedService);
Console.WriteLine(SafeJsonConvert.SerializeObject(
storageLinkedService, client.SerializationSettings));
// Create an Azure Blob dataset
Console.WriteLine("Creating dataset " + blobDatasetName + "...");
DatasetResource blobDataset = new DatasetResource(
new AzureBlobDataset
{
LinkedServiceName = new LinkedServiceReference
{
ReferenceName = storageLinkedServiceName
},
FolderPath = new Expression {
Value = "@{dataset().path}"
},
Parameters = new Dictionary <string, ParameterSpecification>
{
{ "path", new ParameterSpecification {
Type = ParameterType.String
} }
}
}
);
client.Datasets.CreateOrUpdate(
resourceGroup, dataFactoryName, blobDatasetName, blobDataset);
Console.WriteLine(
SafeJsonConvert.SerializeObject(blobDataset, client.SerializationSettings));
// Create a pipeline with a copy activity
Console.WriteLine("Creating pipeline " + pipelineName + "...");
PipelineResource pipeline = new PipelineResource
{
Parameters = new Dictionary <string, ParameterSpecification>
{
{ "inputPath", new ParameterSpecification {
Type = ParameterType.String
} },
{ "outputPath", new ParameterSpecification {
Type = ParameterType.String
} }
},
Activities = new List <Activity>
{
new CopyActivity
{
Name = "CopyFromBlobToBlob",
Inputs = new List <DatasetReference>
{
new DatasetReference()
{
ReferenceName = blobDatasetName,
Parameters = new Dictionary <string, object>
{
{ "path", "@pipeline().parameters.inputPath" }
}
}
},
Outputs = new List <DatasetReference>
{
new DatasetReference
{
ReferenceName = blobDatasetName,
Parameters = new Dictionary <string, object>
{
{ "path", "@pipeline().parameters.outputPath" }
}
}
},
Source = new BlobSource {
},
Sink = new BlobSink {
}
}
}
};
client.Pipelines.CreateOrUpdate(resourceGroup, dataFactoryName, pipelineName, pipeline);
Console.WriteLine(SafeJsonConvert.SerializeObject(pipeline, client.SerializationSettings));
// Create a pipeline run
Console.WriteLine("Creating pipeline run...");
Dictionary <string, object> parameters = new Dictionary <string, object>
{
{ "inputPath", inputBlobPath },
{ "outputPath", outputBlobPath }
};
CreateRunResponse runResponse = client.Pipelines.CreateRunWithHttpMessagesAsync(
resourceGroup, dataFactoryName, pipelineName, parameters: parameters
).Result.Body;
Console.WriteLine("Pipeline run ID: " + runResponse.RunId);
// Monitor the pipeline run
Console.WriteLine("Checking pipeline run status...");
PipelineRun pipelineRun;
while (true)
{
pipelineRun = client.PipelineRuns.Get(
resourceGroup, dataFactoryName, runResponse.RunId);
Console.WriteLine("Status: " + pipelineRun.Status);
if (pipelineRun.Status == "InProgress" || pipelineRun.Status == "Queued")
{
System.Threading.Thread.Sleep(15000);
}
else
{
break;
}
}
// Check the copy activity run details
Console.WriteLine("Checking copy activity run details...");
RunFilterParameters filterParams = new RunFilterParameters(
DateTime.UtcNow.AddMinutes(-10), DateTime.UtcNow.AddMinutes(10));
ActivityRunsQueryResponse queryResponse = client.ActivityRuns.QueryByPipelineRun(
resourceGroup, dataFactoryName, runResponse.RunId, filterParams);
if (pipelineRun.Status == "Succeeded")
{
Console.WriteLine(queryResponse.Value.First().Output);
}
else
{
Console.WriteLine(queryResponse.Value.First().Error);
}
Console.WriteLine("\nPress any key to exit...");
Console.ReadKey();
}
public static async Task <IActionResult> Run(
[HttpTrigger(AuthorizationLevel.Function, "get", "post", Route = null)] HttpRequest req,
ILogger log)
{
log.LogInformation("C# HTTP trigger function processed a request.");
string requestBody = await new StreamReader(req.Body).ReadToEndAsync();
dynamic inputData = JsonConvert.DeserializeObject(requestBody);
string tenantId = inputData?.tenantId;
string applicationId = inputData?.applicationId;
string authenticationKey = inputData?.authenticationKey;
string subscriptionId = inputData?.subscriptionId;
string resourceGroup = inputData?.resourceGroup;
string factoryName = inputData?.factoryName;
string pipelineName = inputData?.pipelineName;
string runId = inputData?.runId;
//Check body for values
if (
tenantId == null ||
applicationId == null ||
authenticationKey == null ||
subscriptionId == null ||
resourceGroup == null ||
factoryName == null ||
pipelineName == null ||
runId == null
)
{
return(new BadRequestObjectResult("Invalid request body, value missing."));
}
//Create a data factory management client
var context = new AuthenticationContext("https://login.windows.net/" + tenantId);
ClientCredential cc = new ClientCredential(applicationId, authenticationKey);
AuthenticationResult result = context.AcquireTokenAsync("https://management.azure.com/", cc).Result;
ServiceClientCredentials cred = new TokenCredentials(result.AccessToken);
var client = new DataFactoryManagementClient(cred)
{
SubscriptionId = subscriptionId
};
//Get pipeline details
int daysOfRuns = 7; //max duration for mandatory RunFilterParameters
DateTime today = DateTime.Now;
DateTime lastWeek = DateTime.Now.AddDays(-daysOfRuns);
PipelineRun pipelineRun;
pipelineRun = client.PipelineRuns.Get(resourceGroup, factoryName, runId);
RunFilterParameters filterParams = new RunFilterParameters(lastWeek, today);
ActivityRunsQueryResponse queryResponse = client.ActivityRuns.QueryByPipelineRun(
resourceGroup, factoryName, runId, filterParams);
//Create initial output content
dynamic outputValues = new JObject();
outputValues.PipelineName = pipelineName;
outputValues.PipelineStatus = pipelineRun.Status;
outputValues.RunId = runId;
outputValues.ResponseCount = queryResponse.Value.Count;
outputValues.ResponseErrorCount = 0;
outputValues.Errors = new JArray();
JObject errorDetails;
log.LogInformation("Pipeline status: " + pipelineRun.Status);
log.LogInformation("Activities found in pipeline response: " + queryResponse.Value.Count.ToString());
//Loop over activities in pipeline run
foreach (var activity in queryResponse.Value)
{
if (String.IsNullOrEmpty(activity.Error.ToString()))
{
continue; //just incase
}
//Parse error output to customise output
dynamic outputData = JsonConvert.DeserializeObject(activity.Error.ToString());
string errorCode = outputData?.errorCode;
string errorType = outputData?.failureType;
string errorMessage = outputData?.message;
//Get output details
if (!String.IsNullOrEmpty(errorCode))
{
log.LogInformation("Activity name: " + activity.ActivityName);
log.LogInformation("Activity type: " + activity.ActivityType);
log.LogInformation("Error message: " + errorMessage);
outputValues.ResponseErrorCount += 1;
//Construct custom error information block
errorDetails = JObject.Parse("{ \"ActivityName\": \"" + activity.ActivityName +
"\", \"ActivityType\": \"" + activity.ActivityType +
"\", \"ErrorCode\": \"" + errorCode +
"\", \"ErrorType\": \"" + errorType +
"\", \"ErrorMessage\": \"" + errorMessage +
"\" }");
outputValues.Errors.Add(errorDetails);
}
}
return(new OkObjectResult(outputValues));
}
public static void QueryActivityRuns(string subscriptionId, string resourceGroup, string factoryName, string runId, string parentRunId, RunFilterParameters filterParameterActivityRuns, Logging logging, ref DataTable dt)
{
#region QueryActivityRuns
logging.LogInformation(String.Format("QueryActivityRuns - RunId {0}", runId));
string outputString = string.Empty;
using (var client = DataFactoryClient.CreateDataFactoryClient(subscriptionId))
{
//Get pipeline status with provided run id
logging.LogInformation(String.Format("QueryActivityRuns - RunId {0} - API QueryByPipelineRun Start", runId));
ActivityRunsQueryResponse activityRunsQueryResponse = client.ActivityRuns.QueryByPipelineRun(resourceGroup, factoryName, runId, filterParameterActivityRuns);
logging.LogInformation(String.Format("QueryActivityRuns - RunId {0} - API QueryByPipelineRun End", runId));
var enumeratorActivity = activityRunsQueryResponse.Value.GetEnumerator();
ActivityRun activityRun;
for (bool hasMoreActivityRuns = enumeratorActivity.MoveNext(); hasMoreActivityRuns;)
{
activityRun = enumeratorActivity.Current;
hasMoreActivityRuns = enumeratorActivity.MoveNext();
DataRow dr = dt.NewRow();
dr["ActivityName"] = activityRun.ActivityName ?? (object)DBNull.Value;
dr["RunId"] = parentRunId;
dr["ActivityRunStart"] = activityRun.ActivityRunStart ?? (object)DBNull.Value;
dr["ActivityRunEnd"] = activityRun.ActivityRunEnd ?? (object)DBNull.Value;
dr["ActivityRunId"] = activityRun.ActivityRunId ?? (object)DBNull.Value;
dr["ActivityType"] = activityRun.ActivityType ?? (object)DBNull.Value;
//dr["AdditionalProperties"] = activityRun.AdditionalProperties ?? (object)DBNull.Value;
dr["DurationInMs"] = activityRun.DurationInMs ?? (object)DBNull.Value;
//dr["Error"] = activityRun.Error ?? (object)DBNull.Value;
//dr["Input"] = activityRun.Input ?? (object)DBNull.Value;
//dr["LinkedServiceName"] = activityRun.LinkedServiceName ?? (object)DBNull.Value;
dr["OutPut"] = activityRun.Output ?? (object)DBNull.Value;
dr["PipelineName"] = activityRun.PipelineName ?? (object)DBNull.Value;
dr["PipelineRunId"] = activityRun.PipelineRunId ?? (object)DBNull.Value;
dr["Status"] = activityRun.Status ?? (object)DBNull.Value;
dt.Rows.Add(dr);
if (activityRun.ActivityType == "ExecutePipeline")
{
string _runId = Shared.JsonHelpers.GetStringValueFromJSON(logging, "pipelineRunId", (JObject)activityRun.Output, null, true);
if (!String.IsNullOrEmpty(_runId))
{
QueryActivityRuns(subscriptionId, resourceGroup, factoryName, _runId, parentRunId, filterParameterActivityRuns, logging, ref dt);
}
else
{
logging.LogInformation(String.Format("QueryActivityRuns - RunId Is null for {0} ", (JObject)activityRun.Output));
}
}
}
}
#endregion
}
static void printActivities(DataFactoryManagementClient client, ActivityRunsQueryResponse activity)
{
var activityEnumerator = activity.Value.GetEnumerator();
ActivityRun activityRun;
while (activityEnumerator.MoveNext())
{
activityRun = activityEnumerator.Current;
//Console.WriteLine("Activity Name:{0}, Start:{1}, End:{2}, Duration(ms):{3}, Status:{4}", activityRun.ActivityName, activityRun.ActivityRunStart.ToString(), activityRun.ActivityRunEnd.ToString(), activityRun.DurationInMs.ToString(), activityRun.Status);
Console.Write(" {0} : {1} ~ {2}({3} ms) - ", activityRun.ActivityName, activityRun.ActivityRunStart.ToString(), activityRun.ActivityRunEnd.ToString(), activityRun.DurationInMs.ToString());
if (activityRun.Status != "Succeeded")
{
Console.BackgroundColor = ConsoleColor.Yellow;
Console.ForegroundColor = ConsoleColor.Red;
}
Console.WriteLine("{0}", activityRun.Status);
Console.BackgroundColor = ConsoleColor.Black;
Console.ForegroundColor = ConsoleColor.White;
if (activityRun.Output != null)
{
JObject jObject = JObject.Parse(activityRun.Output.ToString());
Console.WriteLine(" Data Read : {0}", jObject.SelectToken("dateRead"));
Console.WriteLine(" Files read : {0}", jObject.SelectToken("filesRead"));
Console.WriteLine(" Files written : {0}", jObject.SelectToken("filesWritten"));
Console.WriteLine(" Copy duration : {0}(s)", jObject.SelectToken("copyDuration"));
if (jObject.SelectToken("errors") != null)
{
if (jObject.SelectToken("errors").Count() > 0)
{
JArray jErrorArray = JArray.Parse(jObject.SelectToken("errors").ToString());
foreach (JObject jError in jErrorArray)
{
JObject jErrorObject = JObject.Parse(jError.ToString());
Console.ForegroundColor = ConsoleColor.Red;
Console.WriteLine("Error Code: {0}, Error Message : {1}", jErrorObject.SelectToken("Code").ToString(), jErrorObject.SelectToken("Message").ToString());
Console.ForegroundColor = ConsoleColor.White;
}
}
}
/*jObject.SelectToken("dateRead");
* dataWritten
* filesRead
* filesWritten
* sourcePeakConnections
* sinkPeakConnections
* copyDuration
* throughput
* errors
* effectiveIntegrationRuntime
* usedDataIntegrationUnits
* usedParallelCopies
*/
//Execution Details
JArray executionDetails = (JArray)jObject.SelectToken("executionDetails");
if (executionDetails != null)
{
foreach (JToken executionDetail in executionDetails)
{
Console.WriteLine(" Source Type : {0}", executionDetail.SelectToken("source").SelectToken("type"));
Console.WriteLine(" Sink Type : {0}", executionDetail.SelectToken("sink").SelectToken("type"));
Console.WriteLine(" Status : {0}", executionDetail.SelectToken("status"));
Console.WriteLine(" Duration : {0}", executionDetail.SelectToken("duration"));
Console.WriteLine(" UsedParallelCopies : {0}", executionDetail.SelectToken("usedParallelCopies"));
/*
* Console.WriteLine(executionDetail.SelectToken("sink").SelectToken("type"));
* Console.WriteLine(executionDetail.SelectToken("status"));
* sink.type
* start
* duration
* usedDataIntegrationUnits
* usedParallelCopies
* detailedDurations.queuingDuration
* detailedDurations.transferDuration
*/
}
}
}
}
}
//static void Main(string[] args)
static async System.Threading.Tasks.Task Main(string[] args)
{
// Set variables
string tenantID = "microsoft.onmicrosoft.com";
string User = "******";
string applicationId = "<your application ID>";
string authenticationKey = "<your authentication key for the application>";
string subscriptionId = "b67993e8-0937-4812-af89-520a495da302";
string resourceGroup = "vchmeha";
string region = "West Central US";
string dataFactoryName = "ADFv2HaHu";
string storageAccount = "fromblob";
string storageKey = "gREZ3TbADUcfWJ+ryfxOH7FKrlCKobMHrpS7Z2PPS/gQIsH02LHAGedlqYVpxm/PeWHDJsU/wkBmNz1Q20Rwyg==";
// specify the container and input folder from which all files
// need to be copied to the output folder.
string inputBlobPath = "adftutorial/input";
//specify the contains and output folder where the files are copied
string outputBlobPath = "adftutorial/output";
// name of the Azure Storage linked service, blob dataset, and the pipeline
string storageLinkedServiceName = "AzureStorageLinkedService";
string blobDatasetName = "BlobDataset";
string pipelineName = "Adfv2QuickStartPipeline";
// Authenticate and create a data factory management client
var azureServiceTokenProvider = new AzureServiceTokenProvider();
var token = await azureServiceTokenProvider.GetAccessTokenAsync("https://management.azure.com/");
var cred = new TokenCredentials(token);
var client = new DataFactoryManagementClient(cred)
{
SubscriptionId = subscriptionId
};
// Create a data factory
Console.WriteLine("Creating data factory " + dataFactoryName + "...");
Factory dataFactory = new Factory
{
Location = region,
Identity = new FactoryIdentity()
};
client.Factories.CreateOrUpdate(resourceGroup, dataFactoryName, dataFactory);
Console.WriteLine(
SafeJsonConvert.SerializeObject(dataFactory, client.SerializationSettings));
while (client.Factories.Get(resourceGroup, dataFactoryName).ProvisioningState ==
"PendingCreation")
{
System.Threading.Thread.Sleep(1000);
}
// Create an Azure Storage linked service
Console.WriteLine("Creating linked service " + storageLinkedServiceName + "...");
LinkedServiceResource storageLinkedService = new LinkedServiceResource(
new AzureStorageLinkedService
{
ConnectionString = new SecureString(
"DefaultEndpointsProtocol=https;AccountName=" + storageAccount +
";AccountKey=" + storageKey)
}
);
client.LinkedServices.CreateOrUpdate(
resourceGroup, dataFactoryName, storageLinkedServiceName, storageLinkedService);
Console.WriteLine(SafeJsonConvert.SerializeObject(
storageLinkedService, client.SerializationSettings));
// Create an Azure Blob dataset
Console.WriteLine("Creating dataset " + blobDatasetName + "...");
DatasetResource blobDataset = new DatasetResource(
new AzureBlobDataset
{
LinkedServiceName = new LinkedServiceReference
{
ReferenceName = storageLinkedServiceName
},
FolderPath = new Expression {
Value = "@{dataset().path}"
},
Parameters = new Dictionary <string, ParameterSpecification>
{
{ "path", new ParameterSpecification {
Type = ParameterType.String
} }
}
}
);
client.Datasets.CreateOrUpdate(
resourceGroup, dataFactoryName, blobDatasetName, blobDataset);
Console.WriteLine(
SafeJsonConvert.SerializeObject(blobDataset, client.SerializationSettings));
// Create a pipeline with a copy activity
Console.WriteLine("Creating pipeline " + pipelineName + "...");
PipelineResource pipeline = new PipelineResource
{
Parameters = new Dictionary <string, ParameterSpecification>
{
{ "inputPath", new ParameterSpecification {
Type = ParameterType.String
} },
{ "outputPath", new ParameterSpecification {
Type = ParameterType.String
} }
},
Activities = new List <Activity>
{
new CopyActivity
{
Name = "CopyFromBlobToBlob",
Inputs = new List <DatasetReference>
{
new DatasetReference()
{
ReferenceName = blobDatasetName,
Parameters = new Dictionary <string, object>
{
{ "path", "@pipeline().parameters.inputPath" }
}
}
},
Outputs = new List <DatasetReference>
{
new DatasetReference
{
ReferenceName = blobDatasetName,
Parameters = new Dictionary <string, object>
{
{ "path", "@pipeline().parameters.outputPath" }
}
}
},
Source = new BlobSource {
},
Sink = new BlobSink {
}
}
}
};
client.Pipelines.CreateOrUpdate(resourceGroup, dataFactoryName, pipelineName, pipeline);
Console.WriteLine(SafeJsonConvert.SerializeObject(pipeline, client.SerializationSettings));
// Create a pipeline run
Console.WriteLine("Creating pipeline run...");
Dictionary <string, object> parameters = new Dictionary <string, object>
{
{ "inputPath", inputBlobPath },
{ "outputPath", outputBlobPath }
};
CreateRunResponse runResponse = client.Pipelines.CreateRunWithHttpMessagesAsync(
resourceGroup, dataFactoryName, pipelineName, parameters: parameters
).Result.Body;
Console.WriteLine("Pipeline run ID: " + runResponse.RunId);
// Monitor the pipeline run
Console.WriteLine("Checking pipeline run status...");
PipelineRun pipelineRun;
while (true)
{
pipelineRun = client.PipelineRuns.Get(
resourceGroup, dataFactoryName, runResponse.RunId);
Console.WriteLine("Status: " + pipelineRun.Status);
if (pipelineRun.Status == "InProgress" || pipelineRun.Status == "Queued")
{
System.Threading.Thread.Sleep(15000);
}
else
{
break;
}
}
// Check the copy activity run details
Console.WriteLine("Checking copy activity run details...");
RunFilterParameters filterParams = new RunFilterParameters(
DateTime.UtcNow.AddMinutes(-10), DateTime.UtcNow.AddMinutes(10));
ActivityRunsQueryResponse queryResponse = client.ActivityRuns.QueryByPipelineRun(
resourceGroup, dataFactoryName, runResponse.RunId, filterParams);
if (pipelineRun.Status == "Succeeded")
{
Console.WriteLine(queryResponse.Value.First().Output);
}
else
{
Console.WriteLine(queryResponse.Value.First().Error);
}
Console.WriteLine("\nPress any key to exit...");
Console.ReadKey();
}
public virtual List <PSActivityRun> ListActivityRuns(ActivityRunFilterOptions activityRunFilter)
{
List <PSActivityRun> activityRuns = new List <PSActivityRun>();
var runFilters = new RunFilterParameters()
{
LastUpdatedAfter = activityRunFilter.RunStartedAfter,
LastUpdatedBefore = activityRunFilter.RunStartedBefore,
Filters = new List <RunQueryFilter>(),
OrderBy = new List <RunQueryOrderBy>(),
};
if (activityRunFilter.ActivityName != null)
{
runFilters.Filters.Add(
new RunQueryFilter()
{
Operand = RunQueryFilterOperand.ActivityName,
OperatorProperty = RunQueryFilterOperator.Equals,
Values = new List <string>()
{
activityRunFilter.ActivityName
}
});
}
if (activityRunFilter.Status != null)
{
runFilters.Filters.Add(
new RunQueryFilter()
{
Operand = RunQueryFilterOperand.Status,
OperatorProperty = RunQueryFilterOperator.Equals,
Values = new List <string>()
{
activityRunFilter.Status
}
});
}
ActivityRunsQueryResponse response = this.DataFactoryManagementClient.ActivityRuns.QueryByPipelineRun(
activityRunFilter.ResourceGroupName,
activityRunFilter.DataFactoryName,
activityRunFilter.PipelineRunId,
runFilters);
activityRuns.AddRange(response.Value.Select(ar =>
new PSActivityRun(ar, activityRunFilter.ResourceGroupName, activityRunFilter.DataFactoryName)));
string continuationToken = response.ContinuationToken;
while (!string.IsNullOrWhiteSpace(continuationToken))
{
runFilters.ContinuationToken = continuationToken;
response = this.DataFactoryManagementClient.ActivityRuns.QueryByPipelineRun(activityRunFilter.ResourceGroupName,
activityRunFilter.DataFactoryName, activityRunFilter.PipelineRunId, runFilters);
activityRuns.AddRange(response.Value.Select(ar =>
new PSActivityRun(ar, activityRunFilter.ResourceGroupName, activityRunFilter.DataFactoryName)));
continuationToken = response.ContinuationToken;
}
return(activityRuns);
}
public override PipelineErrorDetail GetPipelineRunActivityErrors(PipelineRunRequest request)
{
PipelineRun pipelineRun = _adfManagementClient.PipelineRuns.Get
(
request.ResourceGroupName,
request.OrchestratorName,
request.RunId
);
//Defensive check
PipelineNameCheck(request.PipelineName, pipelineRun.PipelineName);
_logger.LogInformation("Create pipeline Activity Runs query filters.");
RunFilterParameters filterParams = new RunFilterParameters
(
request.ActivityQueryStart,
request.ActivityQueryEnd
);
_logger.LogInformation("Querying ADF pipeline for Activity Runs.");
ActivityRunsQueryResponse queryResponse = _adfManagementClient.ActivityRuns.QueryByPipelineRun
(
request.ResourceGroupName,
request.OrchestratorName,
request.RunId,
filterParams
);
//Create initial output content
PipelineErrorDetail output = new PipelineErrorDetail()
{
PipelineName = request.PipelineName,
ActualStatus = pipelineRun.Status,
RunId = request.RunId,
ResponseCount = queryResponse.Value.Count
};
_logger.LogInformation("Pipeline status: " + pipelineRun.Status);
_logger.LogInformation("Activities found in pipeline response: " + queryResponse.Value.Count.ToString());
//Loop over activities in pipeline run
foreach (ActivityRun activity in queryResponse.Value)
{
if (activity.Error == null)
{
continue; //only want errors
}
//Parse error output to customise output
dynamic outputBlockInner = JsonConvert.DeserializeObject(activity.Error.ToString());
string errorCode = outputBlockInner?.errorCode;
string errorType = outputBlockInner?.failureType;
string errorMessage = outputBlockInner?.message;
_logger.LogInformation("Activity run id: " + activity.ActivityRunId);
_logger.LogInformation("Activity name: " + activity.ActivityName);
_logger.LogInformation("Activity type: " + activity.ActivityType);
_logger.LogInformation("Error message: " + errorMessage);
output.Errors.Add(new FailedActivity()
{
ActivityRunId = activity.ActivityRunId,
ActivityName = activity.ActivityName,
ActivityType = activity.ActivityType,
ErrorCode = errorCode,
ErrorType = errorType,
ErrorMessage = errorMessage
});
}
return(output);
}
public static async Task <IActionResult> Run(
[HttpTrigger(AuthorizationLevel.Function, "get", "post", Route = null)] HttpRequest req,
ILogger log)
{
log.LogInformation("GetActivityErrors Function triggered by HTTP request.");
#region ParseRequestBody
log.LogInformation("Parsing body from request.");
string requestBody = await new StreamReader(req.Body).ReadToEndAsync();
dynamic inputData = JsonConvert.DeserializeObject(requestBody);
string tenantId = inputData?.tenantId;
string applicationId = inputData?.applicationId;
string authenticationKey = inputData?.authenticationKey;
string subscriptionId = inputData?.subscriptionId;
string resourceGroup = inputData?.resourceGroup;
string factoryName = inputData?.factoryName;
string pipelineName = inputData?.pipelineName;
string runId = inputData?.runId;
//Check body for values
if (
tenantId == null ||
applicationId == null ||
authenticationKey == null ||
subscriptionId == null ||
resourceGroup == null ||
factoryName == null ||
pipelineName == null ||
runId == null
)
{
log.LogInformation("Invalid body.");
return(new BadRequestObjectResult("Invalid request body, value(s) missing."));
}
#endregion
//Query and output support variables
int daysOfRuns = 7; //max duration for mandatory RunFilterParameters
DateTime today = DateTime.Now;
DateTime lastWeek = DateTime.Now.AddDays(-daysOfRuns);
dynamic outputBlock = new JObject();
dynamic outputBlockInner;
//Create a data factory management client
log.LogInformation("Creating ADF connectivity client.");
using (var client = DataFactoryClient.CreateDataFactoryClient(tenantId, applicationId, authenticationKey, subscriptionId))
{
#region SetPipelineRunDetails
//Get pipeline details
PipelineRun pipelineRun;
pipelineRun = client.PipelineRuns.Get(resourceGroup, factoryName, runId);
log.LogInformation("Querying ADF pipeline for Activity Runs.");
RunFilterParameters filterParams = new RunFilterParameters(lastWeek, today);
ActivityRunsQueryResponse queryResponse = client.ActivityRuns.QueryByPipelineRun(resourceGroup, factoryName, runId, filterParams);
//Create initial output content
outputBlock.PipelineName = pipelineName;
outputBlock.PipelineStatus = pipelineRun.Status;
outputBlock.RunId = runId;
outputBlock.ResponseCount = queryResponse.Value.Count;
outputBlock.ResponseErrorCount = 0;
outputBlock.Errors = new JArray();
JObject errorDetails;
#endregion
log.LogInformation("Pipeline status: " + pipelineRun.Status);
log.LogInformation("Activities found in pipeline response: " + queryResponse.Value.Count.ToString());
#region GetActivityDetails
//Loop over activities in pipeline run
foreach (var activity in queryResponse.Value)
{
if (string.IsNullOrEmpty(activity.Error.ToString()))
{
continue; //just incase
}
//Parse error output to customise output
outputBlockInner = JsonConvert.DeserializeObject(activity.Error.ToString());
string errorCode = outputBlockInner?.errorCode;
string errorType = outputBlockInner?.failureType;
string errorMessage = outputBlockInner?.message;
//Get output details
if (!string.IsNullOrEmpty(errorCode))
{
log.LogInformation("Activity run id: " + activity.ActivityRunId);
log.LogInformation("Activity name: " + activity.ActivityName);
log.LogInformation("Activity type: " + activity.ActivityType);
log.LogInformation("Error message: " + errorMessage);
outputBlock.ResponseErrorCount += 1;
//Construct custom error information block
errorDetails = JObject.Parse("{ \"ActivityRunId\": \"" + activity.ActivityRunId +
"\", \"ActivityName\": \"" + activity.ActivityName +
"\", \"ActivityType\": \"" + activity.ActivityType +
"\", \"ErrorCode\": \"" + errorCode +
"\", \"ErrorType\": \"" + errorType +
"\", \"ErrorMessage\": \"" + errorMessage +
"\" }");
outputBlock.Errors.Add(errorDetails);
}
}
#endregion
}
log.LogInformation("GetActivityErrors Function complete.");
return(new OkObjectResult(outputBlock));
}
public PSActivityRunsQueryResponse(ActivityRunsQueryResponse response)
{
this.Value = response?.Value?.Select(element => new PSActivityRun(element)).ToList();
this.ContinuationToken = response?.ContinuationToken;
}
public override PipelineErrorDetail GetPipelineRunActivityErrors(PipelineRunRequest request)
{
PipelineRun pipelineRun = _pipelineRunClient.GetPipelineRun
(
request.RunId
);
//Defensive check
PipelineNameCheck(request.PipelineName, pipelineRun.PipelineName);
_logger.LogInformation("Create pipeline Activity Runs query filters.");
RunFilterParameters filterParams = new RunFilterParameters
(
request.ActivityQueryStart,
request.ActivityQueryEnd
);
_logger.LogInformation("Querying SYN pipeline for Activity Runs.");
ActivityRunsQueryResponse queryResponse = _pipelineRunClient.QueryActivityRuns
(
request.PipelineName,
request.RunId,
filterParams
);
//Create initial output content
PipelineErrorDetail output = new PipelineErrorDetail()
{
PipelineName = request.PipelineName,
ActualStatus = pipelineRun.Status,
RunId = request.RunId,
ResponseCount = queryResponse.Value.Count
};
_logger.LogInformation("Pipeline status: " + pipelineRun.Status);
_logger.LogInformation("Activities found in pipeline response: " + queryResponse.Value.Count.ToString());
//Loop over activities in pipeline run
foreach (ActivityRun activity in queryResponse.Value)
{
if (activity.Error == null)
{
continue; //only want errors
}
//Parse error output to customise output
var json = JsonConvert.SerializeObject(activity.Error);
Dictionary <string, object> errorContent = JsonConvert.DeserializeObject <Dictionary <string, object> >(json);
_logger.LogInformation("Activity run id: " + activity.ActivityRunId);
_logger.LogInformation("Activity name: " + activity.ActivityName);
_logger.LogInformation("Activity type: " + activity.ActivityType);
_logger.LogInformation("Error message: " + errorContent["message"].ToString());
output.Errors.Add(new FailedActivity()
{
ActivityRunId = activity.ActivityRunId,
ActivityName = activity.ActivityName,
ActivityType = activity.ActivityType,
ErrorCode = errorContent["errorCode"].ToString(),
ErrorType = errorContent["failureType"].ToString(),
ErrorMessage = errorContent["message"].ToString()
});
}
return(output);
}
