public static IActionResult ExtractPipelineLog(
            [HttpTrigger(AuthorizationLevel.Anonymous, "get", Route = null)] HttpRequest req,
            ILogger log)
        {
            log.LogInformation("Extracting pipeline log.");
            string pipelineName = req.Query["pipelineName"];

            if (pipelineName == null)
            {
                return(new BadRequestObjectResult("Please pass a pipelineName on the query string"));
            }

            var client = buildClient();

            RunFilterParameters filterParameters = new RunFilterParameters();

            filterParameters.LastUpdatedAfter  = DateTime.MinValue;
            filterParameters.LastUpdatedBefore = DateTime.MaxValue;
            PipelineRunsQueryResponse pipelineRuns = client.PipelineRuns.QueryByFactory(
                getResourceGroup(), getDataFactoryName(), filterParameters);
            PipelineRun latestPipelineRun = null;

            foreach (var run in pipelineRuns.Value)
            {
                if (run.PipelineName == pipelineName &&
                    ((latestPipelineRun == null) || (run.IsLatest == true && run.LastUpdated > latestPipelineRun.LastUpdated)))
                {
                    latestPipelineRun = run;
                }
            }

            return((ActionResult) new OkObjectResult(JsonConvert.SerializeObject(latestPipelineRun)));
        }
Esempio n. 2
0
 public void PipelineRuns_Get()
 {
     RunTest("PipelineRuns_Get", (example, client, responseCode) =>
     {
         PipelineRun response = client.PipelineRuns.Get(RGN(example), FN(example), GetTypedParameter <string>(example, client, "runId"));
         CheckResponseBody(example, client, responseCode, response);
     });
 }
        public PSPipelineRun(PipelineRun pipelineRun, string resourceGroupName, string factoryName)
        {
            if (pipelineRun == null)
            {
                throw new ArgumentNullException("pipelineRun");
            }

            this.pipelineRun       = pipelineRun;
            this.ResourceGroupName = resourceGroupName;
            this.DataFactoryName   = factoryName;
        }
Esempio n. 4
0
        public static void initialise()
        {
            DataFactoryManagementClient client = ADF.GetAuthUser();

            ADF.CreateADF(client);
            ADF.CreateBlobLS(client);
            ADF.CreateDS(client);
            ADF.CreatePipeline(client);
            CreateRunResponse runResponse = ADF.CreatePipelineRun(client);
            PipelineRun       pipelineRun = ADF.MonitorPipelineStatus(client, runResponse);

            ADF.MonitorActivityStatus(client, runResponse, pipelineRun);
        }
        public async Task TestGet()
        {
            PipelineClient    pipelineClient = CreatePipelineClient();
            PipelineRunClient runClient      = CreatePipelineRunClient();

            await using DisposablePipeline pipeline = await DisposablePipeline.Create(pipelineClient, this.Recording);

            CreateRunResponse runResponse = await pipelineClient.CreatePipelineRunAsync(pipeline.Name);

            Assert.NotNull(runResponse.RunId);

            PipelineRun run = await runClient.GetPipelineRunAsync(runResponse.RunId);

            Assert.AreEqual(run.RunId, runResponse.RunId);
            Assert.NotNull(run.Status);
        }
Esempio n. 6
0
 public PSPipelineRun(PipelineRun pipelineRun, string workspaceName)
 {
     this.WorkspaceName        = workspaceName;
     this.Message              = pipelineRun?.Message;
     this.Status               = pipelineRun?.Status;
     this.DurationInMs         = pipelineRun?.DurationInMs;
     this.RunEnd               = pipelineRun?.RunEnd;
     this.RunStart             = pipelineRun?.RunStart;
     this.LastUpdated          = pipelineRun?.LastUpdated;
     this.InvokedBy            = new PSPipelineRunInvokedBy(pipelineRun?.InvokedBy);
     this.Parameters           = pipelineRun?.Parameters;
     this.PipelineName         = pipelineRun?.PipelineName;
     this.IsLatest             = pipelineRun?.IsLatest;
     this.RunGroupId           = pipelineRun?.RunGroupId;
     this.RunId                = pipelineRun?.RunId;
     this.AdditionalProperties = pipelineRun?.AdditionalProperties;
 }
Esempio n. 7
0
 private void CapturePipelineRuns_Get(string runId)
 {
     interceptor.CurrentExampleName = "PipelineRuns_Get";
     PipelineRun run = client.PipelineRuns.Get(secrets.ResourceGroupName, secrets.FactoryName, runId);
 }
Esempio n. 8
0
        static void Main(string[] args)
        {
            //Authentification auprès d'Azure avec l'application svc_adf
            AuthenticationContext       context   = new AuthenticationContext("https://login.windows.net/" + _repertoireId);
            ClientCredential            cc        = new ClientCredential(_applicationId, _applicationKey);
            AuthenticationResult        result    = context.AcquireTokenAsync("https://management.azure.com/", cc).Result;
            ServiceClientCredentials    cred      = new TokenCredentials(result.AccessToken);
            DataFactoryManagementClient ADFclient = new DataFactoryManagementClient(cred)
            {
                SubscriptionId = _abonnementId
            };

            //Création d'une Azure Data Factory

            Factory dataFactory = new Factory
            {
                Location = _region,
                Identity = new FactoryIdentity()
            };

            ADFclient.Factories.CreateOrUpdate(_ressourceGroup, _dataFactoryName, dataFactory);
            Console.WriteLine(SafeJsonConvert.SerializeObject(dataFactory, ADFclient.SerializationSettings));
            var toto = ADFclient.Factories.Get(_ressourceGroup, _dataFactoryName).ProvisioningState;

            while (ADFclient.Factories.Get(_ressourceGroup, _dataFactoryName).ProvisioningState == "PendingCreation")
            {
                System.Threading.Thread.Sleep(1000);
                Console.WriteLine("*");
            }

            ////Création d'un Integration Runtime Auto-Hébergé

            //IntegrationRuntimeResource integrationRuntimeResource = new IntegrationRuntimeResource(
            //new SelfHostedIntegrationRuntime
            //{
            //    Description = "L'Integration Runtime du projet ..."
            //}
            //);
            //ADFclient.IntegrationRuntimes.CreateOrUpdate(_ressourceGroup, _dataFactoryName, _IntegrationRuntimeName, integrationRuntimeResource);
            //Console.WriteLine(SafeJsonConvert.SerializeObject(integrationRuntimeResource, ADFclient.SerializationSettings));
            //Console.WriteLine("Authkey : " + ADFclient.IntegrationRuntimes.ListAuthKeys(_ressourceGroup, _dataFactoryName, _IntegrationRuntimeName).AuthKey1);

            //Création service lié File System on premise

            IntegrationRuntimeReference integrationRuntimeReference = new IntegrationRuntimeReference(_IntegrationRuntimeName);
            SecureString          secureString        = new SecureString("MonPassword");
            LinkedServiceResource FS_PartageOnPremise = new LinkedServiceResource(
                new FileServerLinkedService
            {
                Description = "Service lié référençant un espace partagé dans le réseau privé de l'entreprise",
                ConnectVia  = integrationRuntimeReference,
                Host        = @"\\IRAutoHeberge\Dépôt",
                UserId      = "chsauget",
                Password    = secureString
            }
                );

            ADFclient.LinkedServices.CreateOrUpdate(_ressourceGroup, _dataFactoryName, _FS_PartageOnPremiseName, FS_PartageOnPremise);
            Console.WriteLine(SafeJsonConvert.SerializeObject(FS_PartageOnPremise, ADFclient.SerializationSettings));

            //Création service lié Azure SQLDB

            SecureString SQLsecureString = new SecureString("integrated security=False;encrypt=True;connection timeout=30;data source=adflivre.database.windows.net;initial catalog=advwrks;user id=chsauget;Password=toto");

            LinkedServiceResource SQDB_AdventureWorks = new LinkedServiceResource(
                new AzureSqlDatabaseLinkedService
            {
                Description      = "Service lié référençant un espace partagé dans le réseau privé de l'entreprise",
                ConnectionString = SQLsecureString,
            }
                );

            ADFclient.LinkedServices.CreateOrUpdate(_ressourceGroup, _dataFactoryName, _SQDB_AdventureWorksName, SQDB_AdventureWorks);
            Console.WriteLine(SafeJsonConvert.SerializeObject(SQDB_AdventureWorks, ADFclient.SerializationSettings));

            //Création jeu de données FS_Customer

            DatasetResource FS_Customer = new DatasetResource(
                new FileShareDataset
            {
                LinkedServiceName = new LinkedServiceReference
                {
                    ReferenceName = _FS_PartageOnPremiseName
                }
                , FolderPath = "AdventureWorks CSV"
                , FileName   = "Customer.csv"
                , Format     = new TextFormat
                {
                    ColumnDelimiter  = "\t",
                    RowDelimiter     = "\n",
                    FirstRowAsHeader = false
                }
                , Structure = new List <DatasetDataElement>
                {
                    new DatasetDataElement
                    {
                        Name = "CustomerID",
                        Type = "Int32"
                    },
                    new DatasetDataElement
                    {
                        Name = "PersonID",
                        Type = "Int32"
                    },
                    new DatasetDataElement
                    {
                        Name = "StoreID",
                        Type = "Int32"
                    },
                    new DatasetDataElement
                    {
                        Name = "TerritoryID",
                        Type = "Int32"
                    },
                    new DatasetDataElement
                    {
                        Name = "AccountNumber",
                        Type = "String"
                    },
                    new DatasetDataElement
                    {
                        Name = "rowguid",
                        Type = "String"
                    },
                    new DatasetDataElement
                    {
                        Name = "ModifiedDate",
                        Type = "DateTime"
                    }
                }
            }
                , name: _FS_CustomerName
                );

            ADFclient.Datasets.CreateOrUpdate(_ressourceGroup, _dataFactoryName, _FS_CustomerName, FS_Customer);
            Console.WriteLine(SafeJsonConvert.SerializeObject(FS_Customer, ADFclient.SerializationSettings));

            //Création jeu de données SQDB_Col_Customer

            DatasetResource SQDB_Col_Customer = new DatasetResource(
                new AzureSqlTableDataset
            {
                LinkedServiceName = new LinkedServiceReference
                {
                    ReferenceName = _SQDB_AdventureWorksName
                },
                TableName = "col.Customer"
            }
                , name: _SQDB_Col_CustomerName
                );

            ADFclient.Datasets.CreateOrUpdate(_ressourceGroup, _dataFactoryName, _SQDB_AdventureWorksName, SQDB_Col_Customer);
            Console.WriteLine(SafeJsonConvert.SerializeObject(SQDB_Col_Customer, ADFclient.SerializationSettings));

            //Création de l'activité de copie du fichier Customer
            CopyActivity CustomerCopy = new CopyActivity
            {
                Name = "Copy - Customer"
                ,
                Inputs = new List <DatasetReference> {
                    new DatasetReference()
                    {
                        ReferenceName = _FS_CustomerName
                    }
                }
                ,
                Outputs = new List <DatasetReference> {
                    new DatasetReference()
                    {
                        ReferenceName = _SQDB_Col_CustomerName
                    }
                }
                ,
                Source = new FileSystemSource
                {
                }
                ,
                Sink = new AzureTableSink
                {
                }
            };
            //Création de l'activité de copie du fichier Customer
            PipelineResource PipelineCustomer = new PipelineResource
            {
                Activities = new List <Activity> {
                    CustomerCopy
                }
                , Folder = new PipelineFolder {
                    Name = "AdventureWorks"
                }
            };

            ADFclient.Pipelines.CreateOrUpdate(_ressourceGroup, _dataFactoryName, "Col_Customer", PipelineCustomer);
            Console.WriteLine(SafeJsonConvert.SerializeObject(PipelineCustomer, ADFclient.SerializationSettings));

            //Demander une execution du pipeline
            CreateRunResponse runResponse = ADFclient.Pipelines.CreateRunWithHttpMessagesAsync(_ressourceGroup, _dataFactoryName, "Col_Customer").Result.Body;

            //Contrôler l'execution du pipeline
            PipelineRun run = ADFclient.PipelineRuns.Get(_ressourceGroup, _dataFactoryName, runResponse.RunId);

            while (run.Status == "InProgress")
            {
                run = ADFclient.PipelineRuns.Get(_ressourceGroup, _dataFactoryName, runResponse.RunId);
                Console.WriteLine("Status: " + run.Status);
            }

//Déclencheur Quotidien
            TriggerResource scheduleTrigger = new TriggerResource(
                new ScheduleTrigger
            {
                Pipelines = new List <TriggerPipelineReference> {
                    new TriggerPipelineReference {
                        PipelineReference = new PipelineReference("Col_Customer")
                    }
                }
                ,
                Recurrence = new ScheduleTriggerRecurrence
                {
                    StartTime = DateTime.Parse("2019-03-30T01:00:00Z")
                    ,
                    Frequency = "Day"
                    ,
                    Interval = 1
                }
            }
                , name: "Daily_01h_Schedule"
                );

            ADFclient.Triggers.CreateOrUpdate(_ressourceGroup, _dataFactoryName, "Daily_01h_Schedule", scheduleTrigger);
            Console.WriteLine(SafeJsonConvert.SerializeObject(scheduleTrigger, ADFclient.SerializationSettings));

            ADFclient.Triggers.BeginStart(_ressourceGroup, _dataFactoryName, "Daily_01h_Schedule");
        }
 public PSPipelineRun()
 {
     this.pipelineRun = new PipelineRun();
 }
Esempio n. 10
0
        public static void MonitorActivityStatus(DataFactoryManagementClient client, CreateRunResponse runResponse, PipelineRun pipelineRun)
        {
            Console.WriteLine("Checking copy activity run details...");

            RunFilterParameters filterParams = new RunFilterParameters(
                DateTime.UtcNow.AddMinutes(-10), DateTime.UtcNow.AddMinutes(10));
            ActivityRunsQueryResponse queryResponse = client.ActivityRuns.QueryByPipelineRun(
                Constants.resourceGroup, Constants.dataFactoryName, runResponse.RunId, filterParams);

            if (pipelineRun.Status == "Succeeded")
            {
                Console.WriteLine(queryResponse.Value.First().Output);
            }
            else
            {
                Console.WriteLine(queryResponse.Value.First().Error);
            }
            Console.WriteLine("\nPress any key to exit...");
            Console.ReadKey();
        }
        public virtual PSPipelineRun GetPipelineRun(string resourceGroup, string datafactoryName, string pipelineRunId)
        {
            PipelineRun pipelineRun = this.DataFactoryManagementClient.PipelineRuns.Get(resourceGroup, datafactoryName, pipelineRunId);

            return(new PSPipelineRun(pipelineRun, resourceGroup, datafactoryName));
        }
        public override PipelineErrorDetail GetPipelineRunActivityErrors(PipelineRunRequest request)
        {
            PipelineRun pipelineRun = _adfManagementClient.PipelineRuns.Get
                                      (
                request.ResourceGroupName,
                request.OrchestratorName,
                request.RunId
                                      );

            //Defensive check
            PipelineNameCheck(request.PipelineName, pipelineRun.PipelineName);

            _logger.LogInformation("Create pipeline Activity Runs query filters.");
            RunFilterParameters filterParams = new RunFilterParameters
                                               (
                request.ActivityQueryStart,
                request.ActivityQueryEnd
                                               );

            _logger.LogInformation("Querying ADF pipeline for Activity Runs.");
            ActivityRunsQueryResponse queryResponse = _adfManagementClient.ActivityRuns.QueryByPipelineRun
                                                      (
                request.ResourceGroupName,
                request.OrchestratorName,
                request.RunId,
                filterParams
                                                      );

            //Create initial output content
            PipelineErrorDetail output = new PipelineErrorDetail()
            {
                PipelineName  = request.PipelineName,
                ActualStatus  = pipelineRun.Status,
                RunId         = request.RunId,
                ResponseCount = queryResponse.Value.Count
            };

            _logger.LogInformation("Pipeline status: " + pipelineRun.Status);
            _logger.LogInformation("Activities found in pipeline response: " + queryResponse.Value.Count.ToString());

            //Loop over activities in pipeline run
            foreach (ActivityRun activity in queryResponse.Value)
            {
                if (activity.Error == null)
                {
                    continue; //only want errors
                }

                //Parse error output to customise output
                dynamic outputBlockInner = JsonConvert.DeserializeObject(activity.Error.ToString());
                string  errorCode        = outputBlockInner?.errorCode;
                string  errorType        = outputBlockInner?.failureType;
                string  errorMessage     = outputBlockInner?.message;

                _logger.LogInformation("Activity run id: " + activity.ActivityRunId);
                _logger.LogInformation("Activity name: " + activity.ActivityName);
                _logger.LogInformation("Activity type: " + activity.ActivityType);
                _logger.LogInformation("Error message: " + errorMessage);

                output.Errors.Add(new FailedActivity()
                {
                    ActivityRunId = activity.ActivityRunId,
                    ActivityName  = activity.ActivityName,
                    ActivityType  = activity.ActivityType,
                    ErrorCode     = errorCode,
                    ErrorType     = errorType,
                    ErrorMessage  = errorMessage
                });
            }
            return(output);
        }
Esempio n. 13
0
        public override PipelineErrorDetail GetPipelineRunActivityErrors(PipelineRunRequest request)
        {
            PipelineRun pipelineRun = _pipelineRunClient.GetPipelineRun
                                      (
                request.RunId
                                      );

            //Defensive check
            PipelineNameCheck(request.PipelineName, pipelineRun.PipelineName);

            _logger.LogInformation("Create pipeline Activity Runs query filters.");
            RunFilterParameters filterParams = new RunFilterParameters
                                               (
                request.ActivityQueryStart,
                request.ActivityQueryEnd
                                               );

            _logger.LogInformation("Querying SYN pipeline for Activity Runs.");
            ActivityRunsQueryResponse queryResponse = _pipelineRunClient.QueryActivityRuns
                                                      (
                request.PipelineName,
                request.RunId,
                filterParams
                                                      );

            //Create initial output content
            PipelineErrorDetail output = new PipelineErrorDetail()
            {
                PipelineName  = request.PipelineName,
                ActualStatus  = pipelineRun.Status,
                RunId         = request.RunId,
                ResponseCount = queryResponse.Value.Count
            };

            _logger.LogInformation("Pipeline status: " + pipelineRun.Status);
            _logger.LogInformation("Activities found in pipeline response: " + queryResponse.Value.Count.ToString());

            //Loop over activities in pipeline run
            foreach (ActivityRun activity in queryResponse.Value)
            {
                if (activity.Error == null)
                {
                    continue; //only want errors
                }

                //Parse error output to customise output
                var json = JsonConvert.SerializeObject(activity.Error);
                Dictionary <string, object> errorContent = JsonConvert.DeserializeObject <Dictionary <string, object> >(json);

                _logger.LogInformation("Activity run id: " + activity.ActivityRunId);
                _logger.LogInformation("Activity name: " + activity.ActivityName);
                _logger.LogInformation("Activity type: " + activity.ActivityType);
                _logger.LogInformation("Error message: " + errorContent["message"].ToString());

                output.Errors.Add(new FailedActivity()
                {
                    ActivityRunId = activity.ActivityRunId,
                    ActivityName  = activity.ActivityName,
                    ActivityType  = activity.ActivityType,
                    ErrorCode     = errorContent["errorCode"].ToString(),
                    ErrorType     = errorContent["failureType"].ToString(),
                    ErrorMessage  = errorContent["message"].ToString()
                });
            }
            return(output);
        }