/// <summary>
/// Deserializes the given json into an Hydra OM Pipeline instance, by mocking a get request to
/// exercise the client's deserialization logic.
/// </summary>
/// <param name="json"></param>
/// <returns></returns>
public static Pipeline DeserializePipelineJson(string json)
{
var handler = new MockResourceProviderDelegatingHandler()
{
Json = json
};
var client = DataPipelineManagementClient.GetFakeClient(handler);
string resourceGroupName = Guid.NewGuid().ToString("D");
string dataFactoryName = Guid.NewGuid().ToString("D");
string pipelineName = Guid.NewGuid().ToString("D");
PipelineGetResponse getResponse = client.Pipelines.Get(resourceGroupName, dataFactoryName, pipelineName);
return(getResponse.Pipeline);
}
public static void CreateOrUpdatePipeline(DataFactoryManagementClient client, string resourceGroupName, string dataFactoryName,
string linkedServiceName, string pipelineName, string dsInput, string dsOutput, string sqlQuery,
DateTime recordDateUTC, string activityName, bool isDataDeploy, CopyOnPremSQLToADLAType cpType)
{
if (String.IsNullOrEmpty(sqlQuery))
{
Console.WriteLine("Cannot create pipeline. Empty SQL Query");
return;
}
try
{
DateTime PipelineActivePeriodStartTime = DateTime.Now.Subtract(TimeSpan.FromDays(1000.00));
DateTime PipelineActivePeriodEndTime = PipelineActivePeriodStartTime;
TimeSpan objActivityDelayPolicyAttr = TimeSpan.FromMinutes(Convert.ToDouble(InitialParams.DelayIntervalOfActivity));
string mode = PipelineMode.OneTime;
if ((isDataDeploy))// && (!cpType.ToString().Equals(CopyOnPremSQLToADLAType.All.ToString())))
{
recordDateUTC.AddHours(3);
PipelineActivePeriodStartTime = recordDateUTC;
PipelineActivePeriodEndTime = recordDateUTC.AddYears(100);
mode = PipelineMode.Scheduled;
}
Scheduler objActivityScheduler = new Scheduler();
objActivityScheduler.Interval = Convert.ToUInt16(InitialParams.ActivityFrequencyInterval);
if (isDataDeploy)
{
objActivityScheduler.Offset = TimeSpan.FromMinutes(Convert.ToDouble(InitialParams.OffsetIntervalOfDataSlice));
}
if (InitialParams.SliceType == SliceType.Start && isDataDeploy)
{
objActivityScheduler.Style = SchedulerStyle.StartOfInterval;
}
switch (InitialParams.ActivityFrequencyType)
{
case Frequency.Month:
{
objActivityScheduler.Frequency = SchedulePeriod.Month;
break;
}
case Frequency.Day:
{
objActivityScheduler.Frequency = SchedulePeriod.Day;
break;
}
case Frequency.Hour:
{
objActivityScheduler.Frequency = SchedulePeriod.Hour;
break;
}
case Frequency.Minute:
{
objActivityScheduler.Frequency = SchedulePeriod.Minute;
break;
}
default:
{
objActivityScheduler.Frequency = SchedulePeriod.Day;
break;
}
}
if (client == null)
{
client = CreateManagementClientInstance();
}
Activity activityInPipeline = new Activity()
{
Name = activityName,
Inputs = new List <ActivityInput>()
{
new ActivityInput()
{
Name = dsInput
}
},
Outputs = new List <ActivityOutput>()
{
new ActivityOutput()
{
Name = dsOutput
}
},
TypeProperties = new CopyActivity()
{
Source = new SqlSource()
{
SqlReaderQuery = sqlQuery
},
Sink = new AzureDataLakeStoreSink()
{
WriteBatchSize = 0,
WriteBatchTimeout = TimeSpan.FromMinutes(0)
}
},
Policy = new ActivityPolicy()
{
Timeout = TimeSpan.FromMinutes(3.0),
Delay = objActivityDelayPolicyAttr,
Concurrency = 1,
ExecutionPriorityOrder = ExecutionPriorityOrder.NewestFirst,
LongRetry = 0,
LongRetryInterval = TimeSpan.FromMinutes(0.0),
Retry = 3
},
};
Pipeline pl = null;
try
{
Console.WriteLine("Finding existing pipeline: " + pipelineName + " ... " + activityInPipeline.Name);
PipelineGetResponse respPipelines = client.Pipelines.Get(resourceGroupName, dataFactoryName, pipelineName);
pl = respPipelines.Pipeline;
Console.WriteLine("FIND SUCCESS STATUS: Now updating existing pipeline " + pipelineName + " ... " + activityInPipeline.Name);
pl.Properties.Activities.Add(activityInPipeline);
PipelineCreateOrUpdateParameters plParameters = new PipelineCreateOrUpdateParameters()
{
Pipeline = pl
};
client.Pipelines.CreateOrUpdate(resourceGroupName, dataFactoryName, plParameters);
Console.WriteLine("updated successfully existing pipeline: " + pipelineName + " ... " + activityInPipeline.Name);
}
catch (Exception ex)
{
Console.WriteLine("Could not found any existing pipeline: " + ex.Message);
}
if (pl == null)
{
// Create a pipeline with a copy activity
Console.WriteLine("Creating new pipeline " + pipelineName + " ... " + activityInPipeline.Name);
client.Pipelines.CreateOrUpdate(resourceGroupName, dataFactoryName,
new PipelineCreateOrUpdateParameters()
{
Pipeline = new Pipeline()
{
Name = pipelineName,
Properties = new PipelineProperties()
{
Description = "Pipeline for data transfer from on-premise SC - SQL Server Datamart to Lake Store",
Activities = new List <Activity>()
{
activityInPipeline
},
// Initial value for pipeline's active period. With this, you won't need to set slice status
Start = PipelineActivePeriodStartTime,
End = PipelineActivePeriodEndTime,
IsPaused = false,
PipelineMode = mode,
HubName = cgsHubName,
ExpirationTime = TimeSpan.FromMinutes(0)
},
}
});
// Create a pipeline with a copy activity
Console.WriteLine("created new pipeline " + pipelineName + " ... " + activityInPipeline.Name);
}
} catch (Exception ex)
{
Console.WriteLine("pipeline " + ex.Message);
}
}
