/// <summary> /// Create or update a pipeline instance. /// </summary> /// <param name='operations'> /// Reference to the /// Microsoft.Azure.Management.DataFactories.IPipelineOperations. /// </param> /// <param name='resourceGroupName'> /// Required. The resource group name of the data factory. /// </param> /// <param name='dataFactoryName'> /// Required. A unique data factory instance name. /// </param> /// <param name='parameters'> /// Required. The parameters required to create or update a pipeline. /// </param> /// <returns> /// The create or update pipeline operation response. /// </returns> public static PipelineCreateOrUpdateResponse BeginCreateOrUpdate(this IPipelineOperations operations, string resourceGroupName, string dataFactoryName, PipelineCreateOrUpdateParameters parameters) { return Task.Factory.StartNew((object s) => { return ((IPipelineOperations)s).BeginCreateOrUpdateAsync(resourceGroupName, dataFactoryName, parameters); } , operations, CancellationToken.None, TaskCreationOptions.None, TaskScheduler.Default).Unwrap().GetAwaiter().GetResult(); }
/// <summary> /// Create or update a pipeline instance. /// </summary> /// <param name='operations'> /// Reference to the /// Microsoft.Azure.Management.DataFactories.IPipelineOperations. /// </param> /// <param name='resourceGroupName'> /// Required. The resource group name of the data factory. /// </param> /// <param name='dataFactoryName'> /// Required. A unique data factory instance name. /// </param> /// <param name='parameters'> /// Required. The parameters required to create or update a pipeline. /// </param> /// <returns> /// The create or update pipeline operation response. /// </returns> public static Task<PipelineCreateOrUpdateResponse> BeginCreateOrUpdateAsync( this IPipelineOperations operations, string resourceGroupName, string dataFactoryName, PipelineCreateOrUpdateParameters parameters) { return operations.BeginCreateOrUpdateAsync( resourceGroupName, dataFactoryName, parameters, CancellationToken.None); }
private void CreatePipeline_MapSimilarProducts(DataFactoryManagementClient client) { // Setup Pipeline Parameters var parameters = new PipelineCreateOrUpdateParameters() { Pipeline = new Pipeline() { Name = "MapSimilarProductsPipeline", Properties = new PipelineProperties() { Description = "This is a pipeline to map the similar productids generated as part of Mahout recommendations to the product name", Start = _pipelineStartDate, End = _pipelineEndDate, Activities = new List<Activity>() { new Activity() { Name = "MapSimilarProductsHiveActivity", Description = "Map Similar Hive Activity", Inputs = new List<ActivityInput>() { new ActivityInput("ProductsSimilarityTable") }, Outputs = new List<ActivityOutput>() { new ActivityOutput("ProductsSimilarityOutputTable") }, LinkedServiceName = "HDInsightLinkedService", TypeProperties = new HDInsightHiveActivity() { ScriptPath = "scripts\\selectsimilarproducts.hql", ScriptLinkedService = "StorageLinkedService", Defines = new Dictionary<string, string>() { { "MAHOUTOUTPUT", UpdateParameters("$$Text.Format('wasb://productrec@<account name>.blob.core.windows.net/itemsimilarity/yearno={0:yyyy}/monthno={0:%M}/', SliceStart)") }, { "SIMILARPRODUCTSOUTPUT", UpdateParameters("$$Text.Format('wasb://productrec@<account name>.blob.core.windows.net/itemsimilarityoutput/yearno={0:yyyy}/monthno={0:%M}/', SliceStart)") } }, }, Policy = new ActivityPolicy() { Concurrency = 1, ExecutionPriorityOrder = "NewestFirst", Retry = 0, Timeout = new TimeSpan(0, 1, 0, 0) } } }, } } }; // Create PipeLine client.Pipelines.CreateOrUpdateAsync(Parameters.Tenant.SiteName, Parameters.Tenant.SiteName, parameters).Wait(); }
private void CreatePipeline_PartitionProductUsage(DataFactoryManagementClient client) { // Setup Pipeline Parameters var parameters = new PipelineCreateOrUpdateParameters() { Pipeline = new Pipeline() { Name = "PartitionProductsUsagePipeline", Properties = new PipelineProperties() { Description = "This is a pipeline to prepare the raw Products Usage data for further processing (v1.0)", Start = _pipelineStartDate, End = _pipelineEndDate, Activities = new List<Activity>() { new Activity() { Name = "BlobPartitionHiveActivity", Description = "Blob Partition Hive Activity", Inputs = new List<ActivityInput>() { new ActivityInput("RawProductsUsageTable") }, Outputs = new List<ActivityOutput>() { new ActivityOutput("PartitionedProductsUsageTable") }, LinkedServiceName = "HDInsightLinkedService", TypeProperties = new HDInsightHiveActivity() { ScriptPath = "scripts\\partitionproductusage.hql", ScriptLinkedService = "StorageLinkedService", Defines = new Dictionary<string, string>() { { "RAWINPUT", UpdateParameters("wasb://productrec@<account name>.blob.core.windows.net/rawusageevents/") }, { "PARTITIONEDOUTPUT", UpdateParameters("wasb://productrec@<account name>.blob.core.windows.net/partitionedusageevents/") }, { "Year", "$$Text.Format('{0:yyyy}',SliceStart)" }, { "Month", "$$Text.Format('{0:%M}',SliceStart)" }, { "Day", "$$Text.Format('{0:%d}',SliceStart)" } }, }, Policy = new ActivityPolicy() { Concurrency = 1, ExecutionPriorityOrder = "NewestFirst", Retry = 2, Timeout = new TimeSpan(0, 1, 0, 0) } } }, } } }; // Create PipeLine client.Pipelines.CreateOrUpdateAsync(Parameters.Tenant.SiteName, Parameters.Tenant.SiteName, parameters).Wait(); }
private void CreatePipeline_PrepareMahoutUsage(DataFactoryManagementClient client) { // Setup Pipeline Parameters var parameters = new PipelineCreateOrUpdateParameters() { Pipeline = new Pipeline() { Name = "PrepareMahoutUsagePipeline", Properties = new PipelineProperties() { Description = "This is a pipeline to prepare the data for feeding into the Mahout engine", Start = _pipelineStartDate, End = _pipelineEndDate, Activities = new List<Activity>() { new Activity() { Name = "PrepareMahoutInputHiveActivity", Description = "Prepare Mahout Input Hive Activity", Inputs = new List<ActivityInput>() { new ActivityInput("PartitionedProductsUsageTable") }, Outputs = new List<ActivityOutput>() { new ActivityOutput("MahoutInputProductsUsageTable") }, LinkedServiceName = "HDInsightLinkedService", TypeProperties = new HDInsightHiveActivity() { ScriptPath = "scripts\\preparemahoutinput.hql", ScriptLinkedService = "StorageLinkedService", Defines = new Dictionary<string, string>() { { "MAHOUTINPUT", UpdateParameters("$$Text.Format('wasb://productrec@<account name>.blob.core.windows.net/mahoutinput/yearno={0:yyyy}/monthno={0:%M}/', SliceStart)") }, { "PARTITIONEDOUTPUT", UpdateParameters("$$Text.Format('wasb://productrec@<account name>.blob.core.windows.net/partitionedusageevents/yearno={0:yyyy}/monthno={0:%M}/', SliceStart)") } }, }, Policy = new ActivityPolicy() { Concurrency = 1, ExecutionPriorityOrder = "NewestFirst", Retry = 1, Timeout = new TimeSpan(0, 1, 0, 0) } } }, } } }; // Create PipeLine client.Pipelines.CreateOrUpdateAsync(Parameters.Tenant.SiteName, Parameters.Tenant.SiteName, parameters).Wait(); }
private void CreatePipeline_PrepareSampleData(DataFactoryManagementClient client) { // Setup Pipeline Parameters var parameters = new PipelineCreateOrUpdateParameters() { Pipeline = new Pipeline() { Name = "PrepareSampleDataPipeline", Properties = new PipelineProperties() { Description = "Prepare Sample Data for Personalized Product Recommendation Use Case", Start = _pipelineStartDate, End = _pipelineEndDate, Activities = new List<Activity>() { new Activity() { Name = "PrepareSampleDataActivity", Description = "Prepare Sample Data for Personalized Product Recommendation Use Case", Outputs = new List<ActivityOutput>() { new ActivityOutput("RawProductsUsageTable") }, LinkedServiceName = "HDInsightLinkedService", TypeProperties = new DotNetActivity() { AssemblyName = "ProductRecDataGenerator.dll", EntryPoint = "ProductRecDataGenerator.DataGenerator", PackageLinkedService = "StorageLinkedService", PackageFile = "packages/ProductRecDataGenerator.zip", ExtendedProperties = new Dictionary<string, string>() { { "sliceStart", "$$Text.Format('{0:yyyyMMddHHmm}', Time.AddMinutes(SliceStart, 0))" } } }, Policy = new ActivityPolicy() { Concurrency = 1, ExecutionPriorityOrder = "NewestFirst", Retry = 1, Timeout = new TimeSpan(0, 2, 0, 0) } } }, } } }; // Create PipeLine client.Pipelines.CreateOrUpdateAsync(Parameters.Tenant.SiteName, Parameters.Tenant.SiteName, parameters).Wait(); }
private void CreatePipeline_ProductsRecommenderMahout(DataFactoryManagementClient client) { // Setup Pipeline Parameters var parameters = new PipelineCreateOrUpdateParameters() { Pipeline = new Pipeline() { Name = "ProductsRecommenderMahoutPipeline", Properties = new PipelineProperties() { Description = "Pipeline to Run a Mahout Custom Map Reduce Jar to generate Recommendations.", Start = _pipelineStartDate, End = _pipelineEndDate, Activities = new List<Activity>() { new Activity() { Name = "MahoutActivity", Description = "Custom Map Reduce to generate Mahout result", Inputs = new List<ActivityInput>() { new ActivityInput("MahoutInputProductsUsageTable") }, Outputs = new List<ActivityOutput>() { new ActivityOutput("ProductsRecommendationTable") }, LinkedServiceName = "HDInsightLinkedService", TypeProperties = new HDInsightMapReduceActivity() { ClassName = "org.apache.mahout.cf.taste.hadoop.similarity.item.ItemSimilarityJob", JarFilePath = "jars/mahout/mahout-core-0.9.0.2.1.12.0-2329-job.jar", JarLinkedService = "StorageLinkedService", Arguments = new List<string>() { UpdateParameters("-s"), UpdateParameters("SIMILARITY_COOCCURRENCE"), UpdateParameters("--input"), UpdateParameters("$$Text.Format('wasb://productrec@<account name>.blob.core.windows.net/mahoutinput/yearno={0:yyyy}/monthno={0:%M}/', SliceStart)"), UpdateParameters("--output"), UpdateParameters("$$Text.Format('wasb://productrec@<account name>.blob.core.windows.net/recommendations/yearno={0:yyyy}/monthno={0:%M}/', SliceStart)"), UpdateParameters("--tempDir"), UpdateParameters("$$Text.Format('wasb://productrec@<account name>.blob.core.windows.net/temprecommendationsdir/yearno={0:yyyy}/monthno={0:%M}/', SliceStart)") }, }, Policy = new ActivityPolicy() { Concurrency = 1, ExecutionPriorityOrder = "NewestFirst", Retry = 1, Timeout = new TimeSpan(0, 1, 0, 0) } } }, } } }; // Create PipeLine client.Pipelines.CreateOrUpdateAsync(Parameters.Tenant.SiteName, Parameters.Tenant.SiteName, parameters).Wait(); }
private void CreatePipeline_EgressRecommendedProductsSql(DataFactoryManagementClient client) { // Setup Pipeline Parameters var parameters = new PipelineCreateOrUpdateParameters() { Pipeline = new Pipeline() { Name = "EgressRecommendedProductsSqlPipeline", Properties = new PipelineProperties() { Description = "Egress Recommendations to Sql Azure", Start = _pipelineStartDate, End = _pipelineEndDate, Activities = new List<Activity>() { new Activity() { Name = "EgressSqlAzure", Description = "Move Recommendations to Sql Azure", Inputs = new List<ActivityInput>() { new ActivityInput("ProductsRecommendationOutputTable") }, Outputs = new List<ActivityOutput>() { new ActivityOutput("ProductsRecommendationSQLTable") }, TypeProperties = new CopyActivity() { Source = new BlobSource() { TreatEmptyAsNull = true }, Sink = new SqlSink() { WriteBatchTimeout = new TimeSpan(0, 1, 0, 0) }, }, Policy = new ActivityPolicy() { Concurrency = 1, ExecutionPriorityOrder = "NewestFirst", Retry = 1, Timeout = new TimeSpan(0, 10, 0, 0) } } }, } } }; // Create PipeLine client.Pipelines.CreateOrUpdateAsync(Parameters.Tenant.SiteName, Parameters.Tenant.SiteName, parameters).Wait(); }