/// <summary> /// Deserializes the given json into an Hydra OM Pipeline instance, by mocking a get request to /// exercise the client's deserialization logic. /// </summary> /// <param name="json"></param> /// <returns></returns> public static Pipeline DeserializePipelineJson(string json) { var handler = new MockResourceProviderDelegatingHandler() { Json = json }; var client = DataPipelineManagementClient.GetFakeClient(handler); string resourceGroupName = Guid.NewGuid().ToString("D"); string dataFactoryName = Guid.NewGuid().ToString("D"); string pipelineName = Guid.NewGuid().ToString("D"); PipelineGetResponse getResponse = client.Pipelines.Get(resourceGroupName, dataFactoryName, pipelineName); return(getResponse.Pipeline); }
public static void CreateOrUpdatePipeline(DataFactoryManagementClient client, string resourceGroupName, string dataFactoryName, string linkedServiceName, string pipelineName, string dsInput, string dsOutput, string sqlQuery, DateTime recordDateUTC, string activityName, bool isDataDeploy, CopyOnPremSQLToADLAType cpType) { if (String.IsNullOrEmpty(sqlQuery)) { Console.WriteLine("Cannot create pipeline. Empty SQL Query"); return; } try { DateTime PipelineActivePeriodStartTime = DateTime.Now.Subtract(TimeSpan.FromDays(1000.00)); DateTime PipelineActivePeriodEndTime = PipelineActivePeriodStartTime; TimeSpan objActivityDelayPolicyAttr = TimeSpan.FromMinutes(Convert.ToDouble(InitialParams.DelayIntervalOfActivity)); string mode = PipelineMode.OneTime; if ((isDataDeploy))// && (!cpType.ToString().Equals(CopyOnPremSQLToADLAType.All.ToString()))) { recordDateUTC.AddHours(3); PipelineActivePeriodStartTime = recordDateUTC; PipelineActivePeriodEndTime = recordDateUTC.AddYears(100); mode = PipelineMode.Scheduled; } Scheduler objActivityScheduler = new Scheduler(); objActivityScheduler.Interval = Convert.ToUInt16(InitialParams.ActivityFrequencyInterval); if (isDataDeploy) { objActivityScheduler.Offset = TimeSpan.FromMinutes(Convert.ToDouble(InitialParams.OffsetIntervalOfDataSlice)); } if (InitialParams.SliceType == SliceType.Start && isDataDeploy) { objActivityScheduler.Style = SchedulerStyle.StartOfInterval; } switch (InitialParams.ActivityFrequencyType) { case Frequency.Month: { objActivityScheduler.Frequency = SchedulePeriod.Month; break; } case Frequency.Day: { objActivityScheduler.Frequency = SchedulePeriod.Day; break; } case Frequency.Hour: { objActivityScheduler.Frequency = SchedulePeriod.Hour; break; } case Frequency.Minute: { objActivityScheduler.Frequency = SchedulePeriod.Minute; break; } default: { objActivityScheduler.Frequency = SchedulePeriod.Day; break; } } if (client == null) { client = CreateManagementClientInstance(); } Activity activityInPipeline = new Activity() { Name = activityName, Inputs = new List <ActivityInput>() { new ActivityInput() { Name = dsInput } }, Outputs = new List <ActivityOutput>() { new ActivityOutput() { Name = dsOutput } }, TypeProperties = new CopyActivity() { Source = new SqlSource() { SqlReaderQuery = sqlQuery }, Sink = new AzureDataLakeStoreSink() { WriteBatchSize = 0, WriteBatchTimeout = TimeSpan.FromMinutes(0) } }, Policy = new ActivityPolicy() { Timeout = TimeSpan.FromMinutes(3.0), Delay = objActivityDelayPolicyAttr, Concurrency = 1, ExecutionPriorityOrder = ExecutionPriorityOrder.NewestFirst, LongRetry = 0, LongRetryInterval = TimeSpan.FromMinutes(0.0), Retry = 3 }, }; Pipeline pl = null; try { Console.WriteLine("Finding existing pipeline: " + pipelineName + " ... " + activityInPipeline.Name); PipelineGetResponse respPipelines = client.Pipelines.Get(resourceGroupName, dataFactoryName, pipelineName); pl = respPipelines.Pipeline; Console.WriteLine("FIND SUCCESS STATUS: Now updating existing pipeline " + pipelineName + " ... " + activityInPipeline.Name); pl.Properties.Activities.Add(activityInPipeline); PipelineCreateOrUpdateParameters plParameters = new PipelineCreateOrUpdateParameters() { Pipeline = pl }; client.Pipelines.CreateOrUpdate(resourceGroupName, dataFactoryName, plParameters); Console.WriteLine("updated successfully existing pipeline: " + pipelineName + " ... " + activityInPipeline.Name); } catch (Exception ex) { Console.WriteLine("Could not found any existing pipeline: " + ex.Message); } if (pl == null) { // Create a pipeline with a copy activity Console.WriteLine("Creating new pipeline " + pipelineName + " ... " + activityInPipeline.Name); client.Pipelines.CreateOrUpdate(resourceGroupName, dataFactoryName, new PipelineCreateOrUpdateParameters() { Pipeline = new Pipeline() { Name = pipelineName, Properties = new PipelineProperties() { Description = "Pipeline for data transfer from on-premise SC - SQL Server Datamart to Lake Store", Activities = new List <Activity>() { activityInPipeline }, // Initial value for pipeline's active period. With this, you won't need to set slice status Start = PipelineActivePeriodStartTime, End = PipelineActivePeriodEndTime, IsPaused = false, PipelineMode = mode, HubName = cgsHubName, ExpirationTime = TimeSpan.FromMinutes(0) }, } }); // Create a pipeline with a copy activity Console.WriteLine("created new pipeline " + pipelineName + " ... " + activityInPipeline.Name); } } catch (Exception ex) { Console.WriteLine("pipeline " + ex.Message); } }