/// <summary>
/// Create or update a pipeline instance.
/// </summary>
/// <param name='operations'>
/// Reference to the
/// Microsoft.Azure.Management.DataFactories.IPipelineOperations.
/// </param>
/// <param name='resourceGroupName'>
/// Required. The resource group name of the data factory.
/// </param>
/// <param name='dataFactoryName'>
/// Required. A unique data factory instance name.
/// </param>
/// <param name='parameters'>
/// Required. The parameters required to create or update a pipeline.
/// </param>
/// <returns>
/// The create or update pipeline operation response.
/// </returns>
public static PipelineCreateOrUpdateResponse BeginCreateOrUpdate(this IPipelineOperations operations, string resourceGroupName, string dataFactoryName, PipelineCreateOrUpdateParameters parameters)
{
return Task.Factory.StartNew((object s) =>
{
return ((IPipelineOperations)s).BeginCreateOrUpdateAsync(resourceGroupName, dataFactoryName, parameters);
}
, operations, CancellationToken.None, TaskCreationOptions.None, TaskScheduler.Default).Unwrap().GetAwaiter().GetResult();
}
/// <summary>
/// Create or update a pipeline instance.
/// </summary>
/// <param name='operations'>
/// Reference to the
/// Microsoft.Azure.Management.DataFactories.IPipelineOperations.
/// </param>
/// <param name='resourceGroupName'>
/// Required. The resource group name of the data factory.
/// </param>
/// <param name='dataFactoryName'>
/// Required. A unique data factory instance name.
/// </param>
/// <param name='parameters'>
/// Required. The parameters required to create or update a pipeline.
/// </param>
/// <returns>
/// The create or update pipeline operation response.
/// </returns>
public static Task<PipelineCreateOrUpdateResponse> BeginCreateOrUpdateAsync(
this IPipelineOperations operations,
string resourceGroupName,
string dataFactoryName,
PipelineCreateOrUpdateParameters parameters)
{
return operations.BeginCreateOrUpdateAsync(
resourceGroupName,
dataFactoryName,
parameters,
CancellationToken.None);
}
private void CreatePipeline_MapSimilarProducts(DataFactoryManagementClient client)
{
// Setup Pipeline Parameters
var parameters = new PipelineCreateOrUpdateParameters()
{
Pipeline = new Pipeline()
{
Name = "MapSimilarProductsPipeline",
Properties = new PipelineProperties()
{
Description = "This is a pipeline to map the similar productids generated as part of Mahout recommendations to the product name",
Start = _pipelineStartDate,
End = _pipelineEndDate,
Activities = new List<Activity>()
{
new Activity()
{
Name = "MapSimilarProductsHiveActivity",
Description = "Map Similar Hive Activity",
Inputs = new List<ActivityInput>()
{
new ActivityInput("ProductsSimilarityTable")
},
Outputs = new List<ActivityOutput>()
{
new ActivityOutput("ProductsSimilarityOutputTable")
},
LinkedServiceName = "HDInsightLinkedService",
TypeProperties = new HDInsightHiveActivity()
{
ScriptPath = "scripts\\selectsimilarproducts.hql",
ScriptLinkedService = "StorageLinkedService",
Defines = new Dictionary<string, string>()
{
{ "MAHOUTOUTPUT", UpdateParameters("$$Text.Format('wasb://productrec@<account name>.blob.core.windows.net/itemsimilarity/yearno={0:yyyy}/monthno={0:%M}/', SliceStart)") },
{ "SIMILARPRODUCTSOUTPUT", UpdateParameters("$$Text.Format('wasb://productrec@<account name>.blob.core.windows.net/itemsimilarityoutput/yearno={0:yyyy}/monthno={0:%M}/', SliceStart)") }
},
},
Policy = new ActivityPolicy()
{
Concurrency = 1,
ExecutionPriorityOrder = "NewestFirst",
Retry = 0,
Timeout = new TimeSpan(0, 1, 0, 0)
}
}
},
}
}
};
// Create PipeLine
client.Pipelines.CreateOrUpdateAsync(Parameters.Tenant.SiteName, Parameters.Tenant.SiteName, parameters).Wait();
}
private void CreatePipeline_PartitionProductUsage(DataFactoryManagementClient client)
{
// Setup Pipeline Parameters
var parameters = new PipelineCreateOrUpdateParameters()
{
Pipeline = new Pipeline()
{
Name = "PartitionProductsUsagePipeline",
Properties = new PipelineProperties()
{
Description = "This is a pipeline to prepare the raw Products Usage data for further processing (v1.0)",
Start = _pipelineStartDate,
End = _pipelineEndDate,
Activities = new List<Activity>()
{
new Activity()
{
Name = "BlobPartitionHiveActivity",
Description = "Blob Partition Hive Activity",
Inputs = new List<ActivityInput>()
{
new ActivityInput("RawProductsUsageTable")
},
Outputs = new List<ActivityOutput>()
{
new ActivityOutput("PartitionedProductsUsageTable")
},
LinkedServiceName = "HDInsightLinkedService",
TypeProperties = new HDInsightHiveActivity()
{
ScriptPath = "scripts\\partitionproductusage.hql",
ScriptLinkedService = "StorageLinkedService",
Defines = new Dictionary<string, string>()
{
{ "RAWINPUT", UpdateParameters("wasb://productrec@<account name>.blob.core.windows.net/rawusageevents/") },
{ "PARTITIONEDOUTPUT", UpdateParameters("wasb://productrec@<account name>.blob.core.windows.net/partitionedusageevents/") },
{ "Year", "$$Text.Format('{0:yyyy}',SliceStart)" },
{ "Month", "$$Text.Format('{0:%M}',SliceStart)" },
{ "Day", "$$Text.Format('{0:%d}',SliceStart)" }
},
},
Policy = new ActivityPolicy()
{
Concurrency = 1,
ExecutionPriorityOrder = "NewestFirst",
Retry = 2,
Timeout = new TimeSpan(0, 1, 0, 0)
}
}
},
}
}
};
// Create PipeLine
client.Pipelines.CreateOrUpdateAsync(Parameters.Tenant.SiteName, Parameters.Tenant.SiteName, parameters).Wait();
}
private void CreatePipeline_PrepareMahoutUsage(DataFactoryManagementClient client)
{
// Setup Pipeline Parameters
var parameters = new PipelineCreateOrUpdateParameters()
{
Pipeline = new Pipeline()
{
Name = "PrepareMahoutUsagePipeline",
Properties = new PipelineProperties()
{
Description = "This is a pipeline to prepare the data for feeding into the Mahout engine",
Start = _pipelineStartDate,
End = _pipelineEndDate,
Activities = new List<Activity>()
{
new Activity()
{
Name = "PrepareMahoutInputHiveActivity",
Description = "Prepare Mahout Input Hive Activity",
Inputs = new List<ActivityInput>()
{
new ActivityInput("PartitionedProductsUsageTable")
},
Outputs = new List<ActivityOutput>()
{
new ActivityOutput("MahoutInputProductsUsageTable")
},
LinkedServiceName = "HDInsightLinkedService",
TypeProperties = new HDInsightHiveActivity()
{
ScriptPath = "scripts\\preparemahoutinput.hql",
ScriptLinkedService = "StorageLinkedService",
Defines = new Dictionary<string, string>()
{
{ "MAHOUTINPUT", UpdateParameters("$$Text.Format('wasb://productrec@<account name>.blob.core.windows.net/mahoutinput/yearno={0:yyyy}/monthno={0:%M}/', SliceStart)") },
{ "PARTITIONEDOUTPUT", UpdateParameters("$$Text.Format('wasb://productrec@<account name>.blob.core.windows.net/partitionedusageevents/yearno={0:yyyy}/monthno={0:%M}/', SliceStart)") }
},
},
Policy = new ActivityPolicy()
{
Concurrency = 1,
ExecutionPriorityOrder = "NewestFirst",
Retry = 1,
Timeout = new TimeSpan(0, 1, 0, 0)
}
}
},
}
}
};
// Create PipeLine
client.Pipelines.CreateOrUpdateAsync(Parameters.Tenant.SiteName, Parameters.Tenant.SiteName, parameters).Wait();
}
private void CreatePipeline_PrepareSampleData(DataFactoryManagementClient client)
{
// Setup Pipeline Parameters
var parameters = new PipelineCreateOrUpdateParameters()
{
Pipeline = new Pipeline()
{
Name = "PrepareSampleDataPipeline",
Properties = new PipelineProperties()
{
Description = "Prepare Sample Data for Personalized Product Recommendation Use Case",
Start = _pipelineStartDate,
End = _pipelineEndDate,
Activities = new List<Activity>()
{
new Activity()
{
Name = "PrepareSampleDataActivity",
Description = "Prepare Sample Data for Personalized Product Recommendation Use Case",
Outputs = new List<ActivityOutput>()
{
new ActivityOutput("RawProductsUsageTable")
},
LinkedServiceName = "HDInsightLinkedService",
TypeProperties = new DotNetActivity()
{
AssemblyName = "ProductRecDataGenerator.dll",
EntryPoint = "ProductRecDataGenerator.DataGenerator",
PackageLinkedService = "StorageLinkedService",
PackageFile = "packages/ProductRecDataGenerator.zip",
ExtendedProperties = new Dictionary<string, string>()
{
{ "sliceStart", "$$Text.Format('{0:yyyyMMddHHmm}', Time.AddMinutes(SliceStart, 0))" }
}
},
Policy = new ActivityPolicy()
{
Concurrency = 1,
ExecutionPriorityOrder = "NewestFirst",
Retry = 1,
Timeout = new TimeSpan(0, 2, 0, 0)
}
}
},
}
}
};
// Create PipeLine
client.Pipelines.CreateOrUpdateAsync(Parameters.Tenant.SiteName, Parameters.Tenant.SiteName, parameters).Wait();
}
private void CreatePipeline_ProductsRecommenderMahout(DataFactoryManagementClient client)
{
// Setup Pipeline Parameters
var parameters = new PipelineCreateOrUpdateParameters()
{
Pipeline = new Pipeline()
{
Name = "ProductsRecommenderMahoutPipeline",
Properties = new PipelineProperties()
{
Description = "Pipeline to Run a Mahout Custom Map Reduce Jar to generate Recommendations.",
Start = _pipelineStartDate,
End = _pipelineEndDate,
Activities = new List<Activity>()
{
new Activity()
{
Name = "MahoutActivity",
Description = "Custom Map Reduce to generate Mahout result",
Inputs = new List<ActivityInput>()
{
new ActivityInput("MahoutInputProductsUsageTable")
},
Outputs = new List<ActivityOutput>()
{
new ActivityOutput("ProductsRecommendationTable")
},
LinkedServiceName = "HDInsightLinkedService",
TypeProperties = new HDInsightMapReduceActivity()
{
ClassName = "org.apache.mahout.cf.taste.hadoop.similarity.item.ItemSimilarityJob",
JarFilePath = "jars/mahout/mahout-core-0.9.0.2.1.12.0-2329-job.jar",
JarLinkedService = "StorageLinkedService",
Arguments = new List<string>()
{
UpdateParameters("-s"),
UpdateParameters("SIMILARITY_COOCCURRENCE"),
UpdateParameters("--input"),
UpdateParameters("$$Text.Format('wasb://productrec@<account name>.blob.core.windows.net/mahoutinput/yearno={0:yyyy}/monthno={0:%M}/', SliceStart)"),
UpdateParameters("--output"),
UpdateParameters("$$Text.Format('wasb://productrec@<account name>.blob.core.windows.net/recommendations/yearno={0:yyyy}/monthno={0:%M}/', SliceStart)"),
UpdateParameters("--tempDir"),
UpdateParameters("$$Text.Format('wasb://productrec@<account name>.blob.core.windows.net/temprecommendationsdir/yearno={0:yyyy}/monthno={0:%M}/', SliceStart)")
},
},
Policy = new ActivityPolicy()
{
Concurrency = 1,
ExecutionPriorityOrder = "NewestFirst",
Retry = 1,
Timeout = new TimeSpan(0, 1, 0, 0)
}
}
},
}
}
};
// Create PipeLine
client.Pipelines.CreateOrUpdateAsync(Parameters.Tenant.SiteName, Parameters.Tenant.SiteName, parameters).Wait();
}
private void CreatePipeline_EgressRecommendedProductsSql(DataFactoryManagementClient client)
{
// Setup Pipeline Parameters
var parameters = new PipelineCreateOrUpdateParameters()
{
Pipeline = new Pipeline()
{
Name = "EgressRecommendedProductsSqlPipeline",
Properties = new PipelineProperties()
{
Description = "Egress Recommendations to Sql Azure",
Start = _pipelineStartDate,
End = _pipelineEndDate,
Activities = new List<Activity>()
{
new Activity()
{
Name = "EgressSqlAzure",
Description = "Move Recommendations to Sql Azure",
Inputs = new List<ActivityInput>()
{
new ActivityInput("ProductsRecommendationOutputTable")
},
Outputs = new List<ActivityOutput>()
{
new ActivityOutput("ProductsRecommendationSQLTable")
},
TypeProperties = new CopyActivity()
{
Source = new BlobSource()
{
TreatEmptyAsNull = true
},
Sink = new SqlSink()
{
WriteBatchTimeout = new TimeSpan(0, 1, 0, 0)
},
},
Policy = new ActivityPolicy()
{
Concurrency = 1,
ExecutionPriorityOrder = "NewestFirst",
Retry = 1,
Timeout = new TimeSpan(0, 10, 0, 0)
}
}
},
}
}
};
// Create PipeLine
client.Pipelines.CreateOrUpdateAsync(Parameters.Tenant.SiteName, Parameters.Tenant.SiteName, parameters).Wait();
}
