/// <summary>
/// Create or update a pipeline instance.
/// </summary>
/// <param name='operations'>
/// Reference to the
/// Microsoft.Azure.Management.DataFactories.IPipelineOperations.
/// </param>
/// <param name='resourceGroupName'>
/// Required. The resource group name of the data factory.
/// </param>
/// <param name='dataFactoryName'>
/// Required. A unique data factory instance name.
/// </param>
/// <param name='parameters'>
/// Required. The parameters required to create or update a pipeline.
/// </param>
/// <returns>
/// The create or update pipeline operation response.
/// </returns>
public static Task <PipelineCreateOrUpdateResponse> BeginCreateOrUpdateAsync(
this IPipelineOperations operations,
string resourceGroupName,
string dataFactoryName,
PipelineCreateOrUpdateParameters parameters)
{
return(operations.BeginCreateOrUpdateAsync(
resourceGroupName,
dataFactoryName,
parameters,
CancellationToken.None));
}
/// <summary>
/// Create or update a pipeline instance.
/// </summary>
/// <param name='operations'>
/// Reference to the
/// Microsoft.Azure.Management.DataFactories.IPipelineOperations.
/// </param>
/// <param name='resourceGroupName'>
/// Required. The resource group name of the data factory.
/// </param>
/// <param name='dataFactoryName'>
/// Required. A unique data factory instance name.
/// </param>
/// <param name='parameters'>
/// Required. The parameters required to create or update a pipeline.
/// </param>
/// <returns>
/// The create or update pipeline operation response.
/// </returns>
public static PipelineCreateOrUpdateResponse BeginCreateOrUpdate(
this IPipelineOperations operations,
string resourceGroupName,
string dataFactoryName,
PipelineCreateOrUpdateParameters parameters)
{
return(Task.Factory.StartNew(
s => ((IPipelineOperations)s).BeginCreateOrUpdateAsync(resourceGroupName, dataFactoryName, parameters),
operations,
CancellationToken.None,
TaskCreationOptions.None,
TaskScheduler.Default).Unwrap().GetAwaiter().GetResult());
}
/// <summary>
/// Serializes the given Pipeline into JSON, by mocking a create request to
/// exercise the client's serialization logic.
/// </summary>
/// <param name="item"></param>
/// <returns></returns>
public static string SerializePipelineToJson(Pipeline item)
{
var createParams = new PipelineCreateOrUpdateParameters()
{
Pipeline = item
};
var handler = new MockResourceProviderDelegatingHandler();
var client = DataPipelineManagementClient.GetFakeClient(handler);
string resourceGroupName = Guid.NewGuid().ToString("D");
string dataFactoryName = Guid.NewGuid().ToString("D");
client.Pipelines.BeginCreateOrUpdate(resourceGroupName, dataFactoryName, createParams);
return(handler.Json);
}
private Core.Models.PipelineCreateOrUpdateParameters ValidateAndConvert(PipelineCreateOrUpdateParameters parameters)
{
// Validate
Ensure.IsNotNull(parameters, "parameters");
Ensure.IsNotNull(parameters.Pipeline, "parameters.Pipeline");
this.ValidateObject(parameters.Pipeline);
// Convert
Core.Models.Pipeline internalPipeline = this.Converter.ToCoreType(parameters.Pipeline);
return(new Core.Models.PipelineCreateOrUpdateParameters()
{
Pipeline = internalPipeline
});
}
public async Task <PipelineCreateOrUpdateResponse> CreateOrUpdateAsync(
string resourceGroupName,
string dataFactoryName,
PipelineCreateOrUpdateParameters parameters,
CancellationToken cancellationToken)
{
Core.Models.PipelineCreateOrUpdateParameters internalParameters = this.ValidateAndConvert(parameters);
Core.Models.PipelineCreateOrUpdateResponse response =
await this.Client.InternalClient.Pipelines.CreateOrUpdateAsync(
resourceGroupName,
dataFactoryName,
internalParameters,
cancellationToken);
return(new PipelineCreateOrUpdateResponse(response, this.Client));
}
public static void CreateOrUpdatePipeline(DataFactoryManagementClient client, string resourceGroupName, string dataFactoryName,
string linkedServiceName, string pipelineName, string dsInput, string dsOutput, string sqlQuery,
DateTime recordDateUTC, string activityName, bool isDataDeploy, CopyOnPremSQLToADLAType cpType)
{
if (String.IsNullOrEmpty(sqlQuery))
{
Console.WriteLine("Cannot create pipeline. Empty SQL Query");
return;
}
try
{
DateTime PipelineActivePeriodStartTime = DateTime.Now.Subtract(TimeSpan.FromDays(1000.00));
DateTime PipelineActivePeriodEndTime = PipelineActivePeriodStartTime;
TimeSpan objActivityDelayPolicyAttr = TimeSpan.FromMinutes(Convert.ToDouble(InitialParams.DelayIntervalOfActivity));
string mode = PipelineMode.OneTime;
if ((isDataDeploy))// && (!cpType.ToString().Equals(CopyOnPremSQLToADLAType.All.ToString())))
{
recordDateUTC.AddHours(3);
PipelineActivePeriodStartTime = recordDateUTC;
PipelineActivePeriodEndTime = recordDateUTC.AddYears(100);
mode = PipelineMode.Scheduled;
}
Scheduler objActivityScheduler = new Scheduler();
objActivityScheduler.Interval = Convert.ToUInt16(InitialParams.ActivityFrequencyInterval);
if (isDataDeploy)
{
objActivityScheduler.Offset = TimeSpan.FromMinutes(Convert.ToDouble(InitialParams.OffsetIntervalOfDataSlice));
}
if (InitialParams.SliceType == SliceType.Start && isDataDeploy)
{
objActivityScheduler.Style = SchedulerStyle.StartOfInterval;
}
switch (InitialParams.ActivityFrequencyType)
{
case Frequency.Month:
{
objActivityScheduler.Frequency = SchedulePeriod.Month;
break;
}
case Frequency.Day:
{
objActivityScheduler.Frequency = SchedulePeriod.Day;
break;
}
case Frequency.Hour:
{
objActivityScheduler.Frequency = SchedulePeriod.Hour;
break;
}
case Frequency.Minute:
{
objActivityScheduler.Frequency = SchedulePeriod.Minute;
break;
}
default:
{
objActivityScheduler.Frequency = SchedulePeriod.Day;
break;
}
}
if (client == null)
{
client = CreateManagementClientInstance();
}
Activity activityInPipeline = new Activity()
{
Name = activityName,
Inputs = new List <ActivityInput>()
{
new ActivityInput()
{
Name = dsInput
}
},
Outputs = new List <ActivityOutput>()
{
new ActivityOutput()
{
Name = dsOutput
}
},
TypeProperties = new CopyActivity()
{
Source = new SqlSource()
{
SqlReaderQuery = sqlQuery
},
Sink = new AzureDataLakeStoreSink()
{
WriteBatchSize = 0,
WriteBatchTimeout = TimeSpan.FromMinutes(0)
}
},
Policy = new ActivityPolicy()
{
Timeout = TimeSpan.FromMinutes(3.0),
Delay = objActivityDelayPolicyAttr,
Concurrency = 1,
ExecutionPriorityOrder = ExecutionPriorityOrder.NewestFirst,
LongRetry = 0,
LongRetryInterval = TimeSpan.FromMinutes(0.0),
Retry = 3
},
};
Pipeline pl = null;
try
{
Console.WriteLine("Finding existing pipeline: " + pipelineName + " ... " + activityInPipeline.Name);
PipelineGetResponse respPipelines = client.Pipelines.Get(resourceGroupName, dataFactoryName, pipelineName);
pl = respPipelines.Pipeline;
Console.WriteLine("FIND SUCCESS STATUS: Now updating existing pipeline " + pipelineName + " ... " + activityInPipeline.Name);
pl.Properties.Activities.Add(activityInPipeline);
PipelineCreateOrUpdateParameters plParameters = new PipelineCreateOrUpdateParameters()
{
Pipeline = pl
};
client.Pipelines.CreateOrUpdate(resourceGroupName, dataFactoryName, plParameters);
Console.WriteLine("updated successfully existing pipeline: " + pipelineName + " ... " + activityInPipeline.Name);
}
catch (Exception ex)
{
Console.WriteLine("Could not found any existing pipeline: " + ex.Message);
}
if (pl == null)
{
// Create a pipeline with a copy activity
Console.WriteLine("Creating new pipeline " + pipelineName + " ... " + activityInPipeline.Name);
client.Pipelines.CreateOrUpdate(resourceGroupName, dataFactoryName,
new PipelineCreateOrUpdateParameters()
{
Pipeline = new Pipeline()
{
Name = pipelineName,
Properties = new PipelineProperties()
{
Description = "Pipeline for data transfer from on-premise SC - SQL Server Datamart to Lake Store",
Activities = new List <Activity>()
{
activityInPipeline
},
// Initial value for pipeline's active period. With this, you won't need to set slice status
Start = PipelineActivePeriodStartTime,
End = PipelineActivePeriodEndTime,
IsPaused = false,
PipelineMode = mode,
HubName = cgsHubName,
ExpirationTime = TimeSpan.FromMinutes(0)
},
}
});
// Create a pipeline with a copy activity
Console.WriteLine("created new pipeline " + pipelineName + " ... " + activityInPipeline.Name);
}
} catch (Exception ex)
{
Console.WriteLine("pipeline " + ex.Message);
}
}
