public static HttpResponseMessage Run([HttpTrigger(AuthorizationLevel.Function, "get",
Route = "ProcessTabularModel/{databaseName}/tables/{tableName}/partitions/{partitionName}")]
HttpRequestMessage req,
string databaseName,
string tableName,
string partitionName,
TraceWriter log)
{
log.Info($"Received request to process specific partition in {databaseName}/{tableName}{partitionName} asynchronously.");
try
{
SqlServerAnalysisServerTabular tabularModel = new SqlServerAnalysisServerTabular()
{
ConnectionString = ConfigurationManager.ConnectionStrings["SsasTabularConnection"].ConnectionString,
DatabaseName = databaseName ?? ConfigurationManager.AppSettings["DatabaseName"]
};
tabularModel.ProcessPartition(tableName, partitionName);
}
catch (Exception e)
{
log.Error($"Error processing partition - {databaseName}/{tableName}/{partitionName}: {e.ToString()}", e);
return(req.CreateErrorResponse(HttpStatusCode.InternalServerError, e));
}
var successMessage = $"Successfully processed partition - {databaseName}/{tableName}/{partitionName}";
log.Info(successMessage);
return(req.CreateResponse(HttpStatusCode.OK, new { result = successMessage }));
}
public static async Task <HttpResponseMessage> Run([HttpTrigger(AuthorizationLevel.Function,
"put",
Route = "ProcessTabularModel/{databaseName}/tables/{tableName}/merge")] HttpRequestMessage req,
string databaseName,
string tableName,
TraceWriter log)
{
log.Info("Received request to merge partitions in " + databaseName + "/" + tableName);
try
{
SqlServerAnalysisServerTabular tabularModel = new SqlServerAnalysisServerTabular()
{
ConnectionString = ConfigurationManager.ConnectionStrings["SsasTabularConnection"].ConnectionString,
DatabaseName = databaseName ?? ConfigurationManager.AppSettings["DatabaseName"]
};
//read the request content and de-serialize the JSON payload of new partition information
string mergePartitionInfoSerialized = await req.Content.ReadAsStringAsync();
MergeParitionInfo mergePartitionInfo = JsonConvert.DeserializeObject <MergeParitionInfo>(mergePartitionInfoSerialized);
tabularModel.MergeParitions(tableName, mergePartitionInfo.TargetPartition, mergePartitionInfo.SourcePartitionNames);
}
catch (Exception e)
{
log.Info($"C# HTTP trigger function exception: {e.ToString()}");
return(req.CreateErrorResponse(HttpStatusCode.InternalServerError, e));
}
var successMessage = $"Successfully merged specified partitions in: {databaseName}/{tableName}";
log.Info(successMessage);
return(req.CreateResponse(HttpStatusCode.OK, new { result = successMessage }));
}
public static HttpResponseMessage Run([HttpTrigger(AuthorizationLevel.Function, "post",
Route = "ProcessTabularModel/{databaseName}/tables/{tableName}/repartition/{count}/bydate/{date?}")] HttpRequestMessage req,
string databaseName,
string tableName,
string count,
string date,
TraceWriter log)
{
log.Info($"Received request to auto repartition {databaseName}/{tableName} by months.");
DateTime targetDate = DateTime.TryParse(date, out targetDate) ? targetDate : DateTime.Today;
int maxPartitionCount = Int32.TryParse(count, out maxPartitionCount) ? maxPartitionCount : 180;
log.Info($"Start Date: {targetDate} | Maximum Partitions: {maxPartitionCount}");
try
{
// Determine number of partitions to be created by month based on number of partitions and end date and their properties
List <NewPartitionInfo> newPartitionInfoList = new List <NewPartitionInfo>();
for (int partitionCount = 0;
partitionCount < maxPartitionCount;
partitionCount++)
{
DateTime currentTargetDate = targetDate.AddMonths(-partitionCount);
NewPartitionInfo newPartitionInfo = new NewPartitionInfo()
{
TableName = tableName,
PartitionName = QueryHelper.GeneratePartitionKey(currentTargetDate, PartitionGranularity.Monthly),
SourceQuery = QueryHelper.GetSourceQueryBasedOnDate(tableName, currentTargetDate, PartitionGranularity.Monthly)
};
newPartitionInfoList.Add(newPartitionInfo);
}
// Create the missing partitions in the database
SqlServerAnalysisServerTabular tabularModel = new SqlServerAnalysisServerTabular()
{
ConnectionString = ConfigurationManager.ConnectionStrings["SsasTabularConnection"].ConnectionString,
DatabaseName = databaseName ?? ConfigurationManager.AppSettings["DatabaseName"]
};
if (newPartitionInfoList.Count > 0)
{
tabularModel.CreatePartitions(tableName, newPartitionInfoList.ToArray());
}
}
catch (Exception e)
{
log.Info($"C# HTTP trigger function exception: {e.ToString()}");
return(req.CreateErrorResponse(HttpStatusCode.InternalServerError, e));
}
return(req.CreateResponse(HttpStatusCode.OK,
new { result = $"Repartitioned {databaseName}/{tableName} - {maxPartitionCount} partitions from {targetDate.ToShortDateString()}" }));
}
public static HttpResponseMessage Run(
[HttpTrigger(AuthorizationLevel.Function, "get",
Route = "ProcessTabularModel/{databaseName}/tables/{tableList}")] HttpRequestMessage req,
string databaseName,
string tableList,
TraceWriter log)
{
log.Info($"Received request to process the table - {databaseName}/{tableList}");
try
{
SqlServerAnalysisServerTabular tabularModel = new SqlServerAnalysisServerTabular()
{
ConnectionString = ConfigurationManager.ConnectionStrings["SsasTabularConnection"].ConnectionString,
DatabaseName = databaseName ?? ConfigurationManager.AppSettings["DatabaseName"]
};
log.Info($"Starting table processing on {databaseName}/{tableList}");
if (tableList.Contains(","))
{
log.Info($"Multiple table processing requested.");
var tableNames = tableList.Split(',');
if (tableNames?.Length > 0)
{
log.Info($"Sending request to process {tableNames?.Length} tables in {databaseName}.");
tabularModel.ProcessTables(tableNames);
}
}
else
{
log.Info($"Single table processing requested.");
tabularModel.ProcessTable(tableList);
}
}
catch (Exception e)
{
log.Error($"Error occured processing {databaseName}/{tableList}. Details: {e.ToString()}", e);
return(req.CreateErrorResponse(HttpStatusCode.InternalServerError, e));
}
var successMessage = $"Successfully processed table - {databaseName}/{tableList}";
log.Info(successMessage);
return(req.CreateResponse(HttpStatusCode.OK, new { result = successMessage }));
}
public static void Run([QueueTrigger("%ProcessPartitionQueue%", Connection = "AzureWebJobsStorage")] string myQueueItem,
[Table("%ProcessPartitionStatusTable%", Connection = "AzureWebJobsStorage")] CloudTable statusTable,
TraceWriter log)
{
log.Info($"Received queue trigger to process partition : {myQueueItem}");
QueueMessageProcesssTabular queueMessage = null;
try
{
queueMessage = JsonConvert.DeserializeObject <QueueMessageProcesssTabular>(myQueueItem);
SqlServerAnalysisServerTabular tabularModel = new SqlServerAnalysisServerTabular()
{
ConnectionString = ConfigurationManager.ConnectionStrings["SsasTabularConnection"].ConnectionString,
DatabaseName = queueMessage.Database ?? ConfigurationManager.AppSettings["DatabaseName"]
};
queueMessage.Status = "Running";
queueMessage.ETag = "*";
TableOperation updateOperation = TableOperation.InsertOrReplace(queueMessage);
statusTable.Execute(updateOperation);
tabularModel.ProcessPartition(queueMessage.Tables, queueMessage.Parition);
queueMessage.Status = "Complete";
queueMessage.ETag = "*";
updateOperation = TableOperation.InsertOrReplace(queueMessage);
statusTable.Execute(updateOperation);
}
catch (Exception e)
{
log.Error($"Error occured processing partition - " +
$"{queueMessage?.Database}/{queueMessage?.Tables}/{queueMessage?.Parition} : {e.ToString()}", e);
queueMessage.Status = "Error Processing";
queueMessage.ErrorDetails = e.ToString();
queueMessage.ETag = "*";
TableOperation updateOperation = TableOperation.InsertOrReplace(queueMessage);
statusTable.Execute(updateOperation);
}
log.Info($"Successfully completed partition processing for {queueMessage?.Database}/{queueMessage?.Tables}/{queueMessage?.Parition}");
}
//[FunctionName("ProcessTabular")]
public static void Run([TimerTrigger("0 * * * * *")] TimerInfo myTimer, TraceWriter log)
{
log.Info($"C# Timer trigger function started at: {DateTime.Now}");
try
{
SqlServerAnalysisServerTabular tabularModel = new SqlServerAnalysisServerTabular()
{
ConnectionString = ConfigurationManager.ConnectionStrings["SsasTabularConnection"].ConnectionString,
DatabaseName = ConfigurationManager.AppSettings["DatabaseName"]
};
tabularModel.ProcessModelFull();
}
catch (Exception e)
{
log.Info($"C# Timer trigger function exception: {e.ToString()}");
}
log.Info($"C# Timer trigger function finished at: {DateTime.Now}");
}
public async static Task <HttpResponseMessage> Run([
HttpTrigger(AuthorizationLevel.Function, "post",
Route = "ProcessTabularModel/{databaseName}/tables/{tableName}/partitions/new")] HttpRequestMessage req,
string databaseName,
string tableName,
TraceWriter log)
{
log.Info($"Received request to create new partitions in {databaseName} /{tableName}");
try
{
SqlServerAnalysisServerTabular tabularModel = new SqlServerAnalysisServerTabular()
{
ConnectionString = ConfigurationManager.ConnectionStrings["SsasTabularConnection"].ConnectionString,
DatabaseName = databaseName ?? ConfigurationManager.AppSettings["DatabaseName"]
};
//read the request content and de-serialize the JSON payload of new partition information
string partitionInfoSerialized = await req.Content.ReadAsStringAsync();
NewPartitionInfo[] partitionInfoList = JsonConvert.DeserializeObject <NewPartitionInfo[]>(partitionInfoSerialized);
if (partitionInfoList != null && partitionInfoList.Length > 0)
{
tabularModel.CreatePartitions(tableName, partitionInfoList);
}
}
catch (Exception e)
{
log.Error($"Error occured creating new partitions in {databaseName}/{tableName}: {e.ToString()}", e);
return(req.CreateErrorResponse(HttpStatusCode.InternalServerError, e));
}
var successMessage = $"Created partitions on {databaseName}/{tableName}";
log.Info(successMessage);
return(req.CreateResponse(HttpStatusCode.OK, new { result = successMessage }));
}
public static void Run([QueueTrigger("%ProcessTableQueue%", Connection = "AzureWebJobsStorage")] string myQueueItem,
[Table("%ProcessTableStatusTable%", Connection = "AzureWebJobsStorage")] CloudTable statusTable,
TraceWriter log)
{
log.Info($"Received queue trigger to process table : {myQueueItem}");
QueueMessageProcesssTabular queueMessage = null;
try
{
queueMessage = JsonConvert.DeserializeObject <QueueMessageProcesssTabular>(myQueueItem);
int maximumRetries = int.TryParse(ConfigurationManager.AppSettings["MaximumRetries"], out maximumRetries) ?
maximumRetries : 0;
int waitTimeinSeconds = int.TryParse(ConfigurationManager.AppSettings["WaitTimeInSeconds"], out waitTimeinSeconds) ?
waitTimeinSeconds : 30;
RetryWaitPattern retryWaitPattern = Enum.TryParse <RetryWaitPattern>(ConfigurationManager.AppSettings["RetryWaitPattern"], out retryWaitPattern) ?
retryWaitPattern : RetryWaitPattern.Equal;
SqlServerAnalysisServerTabular tabularModel = new SqlServerAnalysisServerTabular()
{
ConnectionString = ConfigurationManager.ConnectionStrings["SsasTabularConnection"].ConnectionString,
DatabaseName = queueMessage.Database ?? ConfigurationManager.AppSettings["DatabaseName"],
NumberOfRetries = maximumRetries,
WaitPattern = retryWaitPattern,
WaitTimeInSecondsBetweenRetries = waitTimeinSeconds
};
queueMessage.Status = "Running";
queueMessage.ETag = "*";
TableOperation updateOperation = TableOperation.InsertOrReplace(queueMessage);
statusTable.Execute(updateOperation);
log.Info($"Starting table processing on {queueMessage.Database}/{queueMessage.Tables}");
if (queueMessage.Tables.Contains(","))
{
log.Info($"Multiple table processing requested.");
var tableNames = queueMessage.Tables.Split(',');
if (tableNames?.Length > 0)
{
log.Info($"Sending request to process {tableNames?.Length} tables in {queueMessage.Database}.");
tabularModel.ProcessTables(tableNames);
}
}
else
{
log.Info($"Single table processing requested.");
tabularModel.ProcessTable(queueMessage.Tables);
}
queueMessage.Status = "Complete";
queueMessage.ETag = "*";
updateOperation = TableOperation.InsertOrReplace(queueMessage);
statusTable.Execute(updateOperation);
}
catch (Exception e)
{
log.Error($"Error occured processing database table - {queueMessage?.Database}/{queueMessage?.Tables} : {e.ToString()}", e);
queueMessage.Status = "Error Processing";
queueMessage.ErrorDetails = e.ToString();
queueMessage.ETag = "*";
TableOperation updateOperation = TableOperation.InsertOrReplace(queueMessage);
statusTable.Execute(updateOperation);
}
log.Info($"Completed table processing for {queueMessage?.Database}/{queueMessage?.Tables}");
}
