public static HttpResponseMessage Run(
[HttpTrigger(AuthorizationLevel.Function,
"get", "post",
Route = "ProcessTabularModel/processing/status/{operation}/{statusTablePartitionKey}/{trackingId}")] HttpRequestMessage req,
string operation,
string statusTablePartitionKey,
string trackingId,
TraceWriter log)
{
log.Info($"Received request to find status for {operation} process with tracking information: {statusTablePartitionKey}/{trackingId}");
string outputMediaType = ConfigurationManager.AppSettings["ProcessingTrackingOutputMediaType"];
CloudTable statusTable = GetCloudTableByOperation(operation);
if (statusTable != null)
{
try
{
TableOperation retrieveOperation = TableOperation.Retrieve <QueueMessageProcesssTabular>(statusTablePartitionKey, trackingId);
TableResult retrievedResult = statusTable.Execute(retrieveOperation);
if (retrievedResult.Result != null)
{
/* Return result if found */
QueueMessageProcesssTabular processStatus = (QueueMessageProcesssTabular)retrievedResult.Result;
log.Info($"Found status for {operation} process with tracking information: {statusTablePartitionKey}/{trackingId} | Status: {processStatus.Status}");
return(req.CreateResponse(HttpStatusCode.OK, processStatus.ToProcessingTrackingInfo()));
}
else
{
/* Return not found error if tracking info was not found */
log.Info($"Could not find status for {operation} process with tracking information: {statusTablePartitionKey}/{trackingId}");
return(req.CreateResponse(HttpStatusCode.NotFound));
}
}
catch (Exception e)
{
log.Error($"Error occurred retrieving status for {operation} process with tracking information: {statusTablePartitionKey}/{trackingId}", e);
return(req.CreateErrorResponse(HttpStatusCode.InternalServerError, e));
}
}
else
{
/* invalid status table */
var errorMessage = $"Unknown operation - {operation}";
log.Info(errorMessage);
return(req.CreateErrorResponse(HttpStatusCode.BadRequest, errorMessage));
}
}
public static HttpResponseMessage Run([HttpTrigger(AuthorizationLevel.Function, "get",
Route = "ProcessTabularModel/{databaseName}/tables/{tableName}/partitions/{partitionName}/async")]
HttpRequestMessage req,
string databaseName,
string tableName,
string partitionName,
[Queue("%ProcessPartitionQueue%", Connection = "AzureWebJobsStorage")] ICollector <QueueMessageProcesssTabular> queue,
[Table("%ProcessPartitionStatusTable%", Connection = "AzureWebJobsStorage")] ICollector <QueueMessageProcesssTabular> statusTable,
TraceWriter log)
{
log.Info($"Received request queue processing of partition - {databaseName}/{tableName}/{partitionName}");
string outputMediaType = ConfigurationManager.AppSettings["ProcessingTrackingOutputMediaType"];
QueueMessageProcesssTabular queuedMessage = null;
try
{
DateTime enqueuedDateTime = DateTime.UtcNow;
string trackingId = Guid.NewGuid().ToString();
queuedMessage = new QueueMessageProcesssTabular()
{
TrackingId = trackingId,
EnqueuedDateTime = enqueuedDateTime,
Database = databaseName,
Tables = tableName,
TargetDate = DateTime.Now,
Parition = partitionName,
Status = "Queued",
PartitionKey = enqueuedDateTime.ToString("yyyy-MM-dd"),
RowKey = trackingId,
ETag = "*"
};
queue.Add(queuedMessage);
statusTable.Add(queuedMessage);
log.Info($"Successfully queued request to process partition - " +
$"{databaseName}//{tableName}/{partitionName} as {queuedMessage.PartitionKey}/{queuedMessage.RowKey}");
}
catch (Exception e)
{
log.Error($"Error trying queue request to process partition - {databaseName}/{tableName}/{partitionName}. Details : {e.ToString()}", e);
return(req.CreateErrorResponse(HttpStatusCode.InternalServerError, e));
}
return(req.CreateResponse(HttpStatusCode.OK, queuedMessage.ToProcessingTrackingInfo()));
}
public static HttpResponseMessage Run(
[HttpTrigger(AuthorizationLevel.Function, "get",
Route = "ProcessTabularModel/{databaseName}/async")] HttpRequestMessage req,
[Queue("%ProcessModelQueue%", Connection = "AzureWebJobsStorage")] ICollector <QueueMessageProcesssTabular> queue,
[Table("%ProcessModelStatusTable%", Connection = "AzureWebJobsStorage")] ICollector <QueueMessageProcesssTabular> statusTable,
string databaseName,
TraceWriter log)
{
log.Info($"Received request to process the model {databaseName} asynchronously.");
QueueMessageProcesssTabular queuedMessage = null;
try
{
DateTime enqueuedDateTime = DateTime.UtcNow;
string trackingId = Guid.NewGuid().ToString();
queuedMessage = new QueueMessageProcesssTabular()
{
TrackingId = trackingId,
EnqueuedDateTime = enqueuedDateTime,
Database = databaseName,
Tables = null,
TargetDate = DateTime.Now,
Parition = null,
Status = "Queued",
PartitionKey = enqueuedDateTime.ToString("yyyy-MM-dd"),
RowKey = trackingId,
ETag = "*"
};
queue.Add(queuedMessage);
statusTable.Add(queuedMessage);
log.Info($"Successfully queued request to process database - {databaseName} as {queuedMessage.PartitionKey}/{queuedMessage.RowKey}");
}
catch (Exception e)
{
log.Error($"Error occured trying to queue request to process database - {databaseName}. Details : {e.ToString()}", e);
return(req.CreateErrorResponse(HttpStatusCode.InternalServerError, e));
}
return(req.CreateResponse(HttpStatusCode.OK, queuedMessage.ToProcessingTrackingInfo()));
}
public static void Run([QueueTrigger("%ProcessPartitionQueue%", Connection = "AzureWebJobsStorage")] string myQueueItem,
[Table("%ProcessPartitionStatusTable%", Connection = "AzureWebJobsStorage")] CloudTable statusTable,
TraceWriter log)
{
log.Info($"Received queue trigger to process partition : {myQueueItem}");
QueueMessageProcesssTabular queueMessage = null;
try
{
queueMessage = JsonConvert.DeserializeObject <QueueMessageProcesssTabular>(myQueueItem);
SqlServerAnalysisServerTabular tabularModel = new SqlServerAnalysisServerTabular()
{
ConnectionString = ConfigurationManager.ConnectionStrings["SsasTabularConnection"].ConnectionString,
DatabaseName = queueMessage.Database ?? ConfigurationManager.AppSettings["DatabaseName"]
};
queueMessage.Status = "Running";
queueMessage.ETag = "*";
TableOperation updateOperation = TableOperation.InsertOrReplace(queueMessage);
statusTable.Execute(updateOperation);
tabularModel.ProcessPartition(queueMessage.Tables, queueMessage.Parition);
queueMessage.Status = "Complete";
queueMessage.ETag = "*";
updateOperation = TableOperation.InsertOrReplace(queueMessage);
statusTable.Execute(updateOperation);
}
catch (Exception e)
{
log.Error($"Error occured processing partition - " +
$"{queueMessage?.Database}/{queueMessage?.Tables}/{queueMessage?.Parition} : {e.ToString()}", e);
queueMessage.Status = "Error Processing";
queueMessage.ErrorDetails = e.ToString();
queueMessage.ETag = "*";
TableOperation updateOperation = TableOperation.InsertOrReplace(queueMessage);
statusTable.Execute(updateOperation);
}
log.Info($"Successfully completed partition processing for {queueMessage?.Database}/{queueMessage?.Tables}/{queueMessage?.Parition}");
}
public static void Run([QueueTrigger("%ProcessTableQueue%", Connection = "AzureWebJobsStorage")] string myQueueItem,
[Table("%ProcessTableStatusTable%", Connection = "AzureWebJobsStorage")] CloudTable statusTable,
TraceWriter log)
{
log.Info($"Received queue trigger to process table : {myQueueItem}");
QueueMessageProcesssTabular queueMessage = null;
try
{
queueMessage = JsonConvert.DeserializeObject <QueueMessageProcesssTabular>(myQueueItem);
int maximumRetries = int.TryParse(ConfigurationManager.AppSettings["MaximumRetries"], out maximumRetries) ?
maximumRetries : 0;
int waitTimeinSeconds = int.TryParse(ConfigurationManager.AppSettings["WaitTimeInSeconds"], out waitTimeinSeconds) ?
waitTimeinSeconds : 30;
RetryWaitPattern retryWaitPattern = Enum.TryParse <RetryWaitPattern>(ConfigurationManager.AppSettings["RetryWaitPattern"], out retryWaitPattern) ?
retryWaitPattern : RetryWaitPattern.Equal;
SqlServerAnalysisServerTabular tabularModel = new SqlServerAnalysisServerTabular()
{
ConnectionString = ConfigurationManager.ConnectionStrings["SsasTabularConnection"].ConnectionString,
DatabaseName = queueMessage.Database ?? ConfigurationManager.AppSettings["DatabaseName"],
NumberOfRetries = maximumRetries,
WaitPattern = retryWaitPattern,
WaitTimeInSecondsBetweenRetries = waitTimeinSeconds
};
queueMessage.Status = "Running";
queueMessage.ETag = "*";
TableOperation updateOperation = TableOperation.InsertOrReplace(queueMessage);
statusTable.Execute(updateOperation);
log.Info($"Starting table processing on {queueMessage.Database}/{queueMessage.Tables}");
if (queueMessage.Tables.Contains(","))
{
log.Info($"Multiple table processing requested.");
var tableNames = queueMessage.Tables.Split(',');
if (tableNames?.Length > 0)
{
log.Info($"Sending request to process {tableNames?.Length} tables in {queueMessage.Database}.");
tabularModel.ProcessTables(tableNames);
}
}
else
{
log.Info($"Single table processing requested.");
tabularModel.ProcessTable(queueMessage.Tables);
}
queueMessage.Status = "Complete";
queueMessage.ETag = "*";
updateOperation = TableOperation.InsertOrReplace(queueMessage);
statusTable.Execute(updateOperation);
}
catch (Exception e)
{
log.Error($"Error occured processing database table - {queueMessage?.Database}/{queueMessage?.Tables} : {e.ToString()}", e);
queueMessage.Status = "Error Processing";
queueMessage.ErrorDetails = e.ToString();
queueMessage.ETag = "*";
TableOperation updateOperation = TableOperation.InsertOrReplace(queueMessage);
statusTable.Execute(updateOperation);
}
log.Info($"Completed table processing for {queueMessage?.Database}/{queueMessage?.Tables}");
}
