public static void Run([TimerTrigger("0 */5 * * * *")] TimerInfo myTimer, ILogger log)
{
try
{
log.LogInformation($"ProcessLandingDataUsingADF Timer trigger function executed at: {DateTime.Now}");
string cloudAccountName = null;
string cloudKey = null;
string queueName = "fileevent";
AzureServiceTokenProvider azureServiceTokenProvider = new AzureServiceTokenProvider();
KeyVaultClient keyvaultClient = new KeyVaultClient(new KeyVaultClient.AuthenticationCallback(azureServiceTokenProvider.KeyVaultTokenCallback));
string keyVault = Environment.GetEnvironmentVariable("MY_KEY_VAULT");
var secretCloudAccountName = keyvaultClient.GetSecretAsync($"https://{keyVault}.vault.azure.net/", "LandingZoneStorageAccountName");
var secretcloudKey = keyvaultClient.GetSecretAsync($"https://{keyVault}.vault.azure.net/", "LandingZoneStorageAccountKey");
cloudAccountName = secretCloudAccountName.Result.Value;
cloudKey = secretcloudKey.Result.Value;
Microsoft.WindowsAzure.Storage.Queue.CloudQueue cloudQueue = QueueService.ConnectToQueueStorage(queueName, cloudAccountName, cloudKey);
Microsoft.WindowsAzure.Storage.Queue.CloudQueueMessage queueItem = QueueService.GetQueueItem(queueName, cloudQueue);
while (queueItem != null)
{
ProcessItem(queueItem, queueName, cloudQueue, log);
queueItem = QueueService.GetQueueItem(queueName, cloudQueue);
}
}
catch (Exception ex)
{
log.LogError("ProcessLandingDataUsingADF Exception: " + ex.ToString());
}
} // Run
} // DeleteQueueItem
/// <summary>
/// Creates the storage and gets a reference (once)
/// </summary>
public static Microsoft.WindowsAzure.Storage.Queue.CloudQueue ConnectToQueueStorage(string queueName, string cloudAccountName, string cloudKey)
{
try
{
Microsoft.WindowsAzure.Storage.Auth.StorageCredentials storageCredentials = new Microsoft.WindowsAzure.Storage.Auth.StorageCredentials(
cloudAccountName, cloudKey);
Microsoft.WindowsAzure.Storage.CloudStorageAccount storageAccount = new Microsoft.WindowsAzure.Storage.CloudStorageAccount(storageCredentials, true);
Microsoft.WindowsAzure.Storage.Queue.CloudQueueClient queueStorage = storageAccount.CreateCloudQueueClient();
queueStorage.DefaultRequestOptions.RetryPolicy = new Microsoft.WindowsAzure.Storage.RetryPolicies.LinearRetry(TimeSpan.FromSeconds(DEFAULT_SAVE_QUEUE_RETRY_WAIT_IN_MILLISECONDS), DEFAULT_SAVE_QUEUE_RETRY_ATTEMPTS);
queueStorage.DefaultRequestOptions.ServerTimeout = new TimeSpan(0, DEFAULT_SAVE_AND_READ_QUEUE_TIMEOUT_IN_MINUTES, 0);
Microsoft.WindowsAzure.Storage.Queue.CloudQueue cloudQueue = queueStorage.GetQueueReference(queueName);
cloudQueue.CreateIfNotExistsAsync();
return(cloudQueue);
}
catch (Exception ex)
{
throw new Exception("Storage services initialization failure. "
+ "Check your storage account configuration settings. If running locally, "
+ "ensure that the Development Storage service is running. \n"
+ ex.Message);
}
} // ConnectToQueueStorage
static BsmController()
{
Trace.TraceInformation("[TRACE] Entering BsmController::BsmController() static initializer...");
// NOTE: Need to fully qualify System.Configuration to disambiguate from ApiController.Configuration
//string strStorageAccountConnectionString =
// System.Configuration.ConfigurationManager.AppSettings["StorageAccountConnectionString"];
string strStorageAccountConnectionString =
Microsoft.WindowsAzure.CloudConfigurationManager.GetSetting("StorageAccountConnectionString");
if (strStorageAccountConnectionString == null)
{
Trace.TraceError("Unable to retrieve storage account connection string");
}
else if (strStorageAccountConnectionString.Length <= 0)
{
Trace.TraceError("Storage account connection string empty");
}
else //connect to the cloud storage account
{
try
{
srStorageAccount = Microsoft.WindowsAzure.Storage.CloudStorageAccount.Parse(strStorageAccountConnectionString);
}
catch (Exception e)
{
Trace.TraceError("Exception occurred when parsing storage account connection string\n{0}\n{1}",
strStorageAccountConnectionString, e.Message);
}
if (srStorageAccount != null)
{
srCloudQueueClient = srStorageAccount.CreateCloudQueueClient();
srBsmQueue = srCloudQueueClient.GetQueueReference(srBsmQueueName);
try
{
if (srBsmQueue.CreateIfNotExists())
{
Trace.TraceInformation("Created Azure BSM queue '{0}'", srBsmQueueName);
}
else
{
Trace.TraceInformation("Got reference to existing BSM queue '{0}'", srBsmQueueName);
}
}
catch (Exception e)
{
Trace.TraceError("Exception occurred when creating queue for inbound BSM bundles\n{0}",
e.Message);
srBsmQueue = null;
}
}
}
Trace.TraceInformation("[TRACE] Exiting BsmController::BsmController() static initializer...");
return;
}
} // GetQueueItem
/// <summary>
/// Removes a message from the queue
/// </summary>
/// <param name="queueName"></param>
/// <param name="cloudQueueMessage"></param>
public static void DeleteQueueItem(string queueName, Microsoft.WindowsAzure.Storage.Queue.CloudQueueMessage cloudQueueMessage,
Microsoft.WindowsAzure.Storage.Queue.CloudQueue cloudQueue)
{
string containerName = queueName.ToLower(); // must be lower case!
cloudQueue.DeleteMessageAsync(cloudQueueMessage);
} // DeleteQueueItem
} // PutQueueItem
/// <summary>
/// Creates the storage and gets a reference (once)
/// </summary>
private static void InitializeStorage(string queueContainer)
{
string containerName = queueContainer.ToString().ToLower(); // must be lower case!
if (storageInitializedDictionary.ContainsKey(containerName) && storageInitializedDictionary[containerName] == true)
{
return;
}
lock (gate)
{
if (storageInitializedDictionary.ContainsKey(containerName) && storageInitializedDictionary[containerName] == true)
{
return;
}
try
{
Microsoft.WindowsAzure.Storage.Auth.StorageCredentials storageCredentials = new Microsoft.WindowsAzure.Storage.Auth.StorageCredentials(
Sample.Azure.Common.Setting.SettingService.CloudStorageAccountName,
Sample.Azure.Common.Setting.SettingService.CloudStorageKey);
Microsoft.WindowsAzure.Storage.CloudStorageAccount storageAccount = new Microsoft.WindowsAzure.Storage.CloudStorageAccount(storageCredentials,
new Uri(Sample.Azure.Common.Setting.SettingService.CloudStorageBlobEndPoint),
new Uri(Sample.Azure.Common.Setting.SettingService.CloudStorageQueueEndPoint),
new Uri(Sample.Azure.Common.Setting.SettingService.CloudStorageTableEndPoint),
new Uri(Sample.Azure.Common.Setting.SettingService.CloudStorageFileEndPoint));
Microsoft.WindowsAzure.Storage.Queue.CloudQueueClient queueStorage = storageAccount.CreateCloudQueueClient();
queueStorage.DefaultRequestOptions.RetryPolicy = new Microsoft.WindowsAzure.Storage.RetryPolicies.LinearRetry(TimeSpan.FromSeconds(DEFAULT_SAVE_QUEUE_RETRY_WAIT_IN_MILLISECONDS), DEFAULT_SAVE_QUEUE_RETRY_ATTEMPTS);
int queueSaveTimeoutInMinutes = DEFAULT_SAVE_AND_READ_QUEUE_TIMEOUT_IN_MINUTES;
string timeOutOverRide = Sample.Azure.Common.Setting.SettingService.SaveAndReadQueueTimeoutInMinutes;
if (timeOutOverRide != null)
{
queueSaveTimeoutInMinutes = int.Parse(timeOutOverRide);
}
queueStorage.DefaultRequestOptions.ServerTimeout = TimeSpan.FromMinutes(queueSaveTimeoutInMinutes);
queueStorage.DefaultRequestOptions.ServerTimeout = new TimeSpan(0, DEFAULT_SAVE_AND_READ_QUEUE_TIMEOUT_IN_MINUTES, 0);
Microsoft.WindowsAzure.Storage.Queue.CloudQueue cloudQueue = queueStorage.GetQueueReference(containerName);
cloudQueue.CreateIfNotExists();
queueStorageDictionary.Add(containerName, queueStorage);
storageInitializedDictionary.Add(containerName, true);
}
catch (Exception ex)
{
throw new Exception("Storage services initialization failure. "
+ "Check your storage account configuration settings. If running locally, "
+ "ensure that the Development Storage service is running. \n"
+ ex.Message);
}
} // lock
} // InitializeStorage
} // DeleteQueueItem
public void DeleteQueueItem(string queueContainer, string messageId, string popReceipt)
{
InitializeStorage(queueContainer);
string containerName = queueContainer.ToString().ToLower(); // must be lower case!
Microsoft.WindowsAzure.Storage.Queue.CloudQueueClient queueStorage = queueStorageDictionary[containerName];
Microsoft.WindowsAzure.Storage.Queue.CloudQueue cloudQueue = queueStorage.GetQueueReference(containerName);
cloudQueue.DeleteMessage(messageId, popReceipt);
} // DeleteQueueItem
} // PutQueueItem
/// <summary>
/// Puts a byte[] message
/// </summary>
/// <param name="queueContainer"></param>
/// <param name="message"></param>
public void PutQueueItem(string queueContainer, MemoryStream message)
{
InitializeStorage(queueContainer);
string containerName = queueContainer.ToString().ToLower(); // must be lower case!
Microsoft.WindowsAzure.Storage.Queue.CloudQueueClient queueStorage = queueStorageDictionary[containerName];
Microsoft.WindowsAzure.Storage.Queue.CloudQueue cloudQueue = queueStorage.GetQueueReference(containerName);
cloudQueue.AddMessage(new Microsoft.WindowsAzure.Storage.Queue.CloudQueueMessage(message.ToArray()));
} // PutQueueItem
} // DeleteQueueItem
/// <summary>
/// Returns the 1st message as a queue item
/// </summary>
/// <param name="queueContainer"></param>
/// <returns></returns>
public Microsoft.WindowsAzure.Storage.Queue.CloudQueueMessage GetQueueItem(string queueContainer)
{
InitializeStorage(queueContainer);
string containerName = queueContainer.ToString().ToLower(); // must be lower case!
Microsoft.WindowsAzure.Storage.Queue.CloudQueueClient queueStorage = queueStorageDictionary[containerName];
Microsoft.WindowsAzure.Storage.Queue.CloudQueue cloudQueue = queueStorage.GetQueueReference(containerName);
Microsoft.WindowsAzure.Storage.Queue.CloudQueueMessage cloudQueueMessage = cloudQueue.GetMessage(TimeSpan.FromMinutes(1));
return(cloudQueueMessage);
} // GetQueueItem
/*
* Member Methods
*/
public override bool OnStart()
{
Microsoft.WindowsAzure.ServiceRuntime.RoleEnvironment.Changed += RoleEnvironment_Changed;
Trace.TraceInformation("[TRACE] Entering BsmWorkerRole::OnStart()...");
ServicePointManager.DefaultConnectionLimit = 12; //maximum number of concurrent connections
string strStorageAccountConnectionString =
Microsoft.WindowsAzure.CloudConfigurationManager.GetSetting("StorageAccountConnectionString");
if (strStorageAccountConnectionString == null)
{
Trace.TraceError("Unable to retrieve storage account connection string");
}
else if (strStorageAccountConnectionString.Length <= 0)
{
Trace.TraceError("Storage account connection string empty");
}
else //connect to the cloud storage account
{
Trace.TraceInformation("[INFO] Retrieved StorageAccountConnectionString for BsmWorkerRole\n{0}",
strStorageAccountConnectionString);
try
{
srStorageAccount = Microsoft.WindowsAzure.Storage.CloudStorageAccount.Parse(strStorageAccountConnectionString);
}
catch (Exception e)
{
Trace.TraceError("Exception occurred when parsing storage account connection string\n{0}\n{1}",
strStorageAccountConnectionString, e.Message);
}
if (srStorageAccount != null)
{
srCloudQueueClient = srStorageAccount.CreateCloudQueueClient();
srBsmQueue = srCloudQueueClient.GetQueueReference(srBsmQueueName);
try
{
if (srBsmQueue.CreateIfNotExists())
{
Trace.TraceInformation("Created Azure BSM queue '{0}'", srBsmQueueName);
}
else
{
Trace.TraceInformation("Got reference to existing BSM queue '{0}'", srBsmQueueName);
}
}
catch (Exception e)
{
Trace.TraceError("Exception occurred when creating queue for inbound BSM bundles\n{0}",
e.Message);
srBsmQueue = null;
}
BsmTimeTableLogger.Initialize(srStorageAccount, srBsmTimeTableName);
}
}
string strDatabaseConnectionString =
Microsoft.WindowsAzure.CloudConfigurationManager.GetSetting("InfloDatabaseConnectionString");
if (strDatabaseConnectionString == null)
{
Trace.TraceError("Unable to retrieve database connection string");
}
else if (strDatabaseConnectionString.Length <= 0)
{
Trace.TraceError("Database connection string empty");
}
else //connect to the database
{
Trace.TraceInformation("[INFO] Retrieved InfloDatabaseConnectionString for BsmWorkerRole\n{0}",
strDatabaseConnectionString);
srDbContext = new InfloDbContext(strDatabaseConnectionString);
}
BsmTimeTableLogger.Enabled = IsStatisticalLoggingEnabled();
BsmTimeTableLogger.MinimalLoggedElapsedTime = GetMinimalLoggedElapsedTime();
Trace.TraceInformation("[TRACE] Exiting BsmWorkerRole::OnStart()...");
return(base.OnStart());
}
} // Run
static void ProcessItem(Microsoft.WindowsAzure.Storage.Queue.CloudQueueMessage queueItem, string queueName, Microsoft.WindowsAzure.Storage.Queue.CloudQueue cloudQueue, ILogger log)
{
log.LogInformation("ProcessItem:" + queueItem);
JSONService jsonService = new JSONService();
FileEvent fileEvent = jsonService.Deserialize <FileEvent>(queueItem.AsString);
log.LogInformation("ProcessItem Topic: " + fileEvent.topic);
log.LogInformation("ProcessItem Subject: " + fileEvent.subject);
// Make sure we have the correct event
if (fileEvent.eventType == "Microsoft.Storage.BlobCreated") // && fileEvent.subject.ToLower().EndsWith("end_file.txt"))
{
// e.g. "subject": "/blobServices/default/containers/acmeinc/blobs/inbox/2020-04-13/test_end_file.txt"
// 01234567890123456789012345678901234567890123456789 14 -> 30 =16
// e.g. "subject": "/blobServices/default/containers/acmeinc/blobs/test_end_file.txt"
// 01234567890123456789012345678901234567890123456789 13 -> 13 = 0
string removedPrefix = fileEvent.subject.ToLower().Replace("/blobservices/default/containers/", string.Empty);
string customerId = removedPrefix.Substring(0, removedPrefix.IndexOf("/"));
string partitionId = removedPrefix.Substring(0, removedPrefix.IndexOf("/")); // this just happens to be the same as the customer id in the this example
// Get the Cosmos DB data as to which pipeline belongs to this customer.
DocumentDBRepository <CosmosIngestionData> documentDBRepository = new DocumentDBRepository <CosmosIngestionData>(log);
var result = documentDBRepository.GetItems(o => o.CustomerId == customerId && o.PartitionId == partitionId).FirstOrDefault();
if (result == null)
{
// Someone forgot to create the CosmosDB document (whoops), send an email and log. Give the developer "1" hour to get things setup.
log.LogInformation("Pipeline does not exist. Setting queue item to retry in 1 hour.");
QueueService.IncreaseQueueItemLock(queueName, queueItem, TimeSpan.FromHours(1), cloudQueue);
}
else
{
if (result.isADFEnabled)
{
// We are good to try to start the ADF
log.LogInformation("Starting Pipeline: " + result.ADFPipelineName);
try
{
// Try to start the pipeline (we are "trying" since technically something could have just disabled it :) )
DateTime dtNow = DateTime.UtcNow; // you could make this EST or such....
string inputFileSystem = customerId;
int blobsIndex = removedPrefix.IndexOf("/blobs/") + 7;
int lastSlashIndex = removedPrefix.LastIndexOf("/");
string inputFileDirectory = removedPrefix.Substring(blobsIndex, lastSlashIndex - blobsIndex);
string outputFileSystem = Environment.GetEnvironmentVariable("LandingZoneDataLakeContainer");
string outputFileDirectory = string.Format("customer-landing-data/{0}/{1:yyyy}/{2:MM}/{3:dd}", customerId, dtNow, dtNow, dtNow);
DataFactoryService.StartPipeline(inputFileSystem, inputFileDirectory, outputFileSystem, outputFileDirectory, result.ADFPipelineName, result.ADFResourceGroup, result.ADFDataFactoryName, result.ADFSubscriptionId, log);
QueueService.DeleteQueueItem(queueName, queueItem, cloudQueue);
}
catch (Exception ex)
{
// Pipeline could not start (it might be in a "provisioning state", set the queue item to try again in "5" minutes
log.LogInformation("Error starting Pipeline (item not dequeued): " + result.ADFPipelineName + " Error: " + ex.ToString());
QueueService.IncreaseQueueItemLock(queueName, queueItem, TimeSpan.FromMinutes(5), cloudQueue);
}
}
else
{
// Pipeline is not enabled, set the queue item to try again in "5" minutes
log.LogInformation("Pipeline: " + result.ADFPipelineName + " is not enabled. Setting queue item to retry in 5 minutes.");
QueueService.IncreaseQueueItemLock(queueName, queueItem, TimeSpan.FromMinutes(5), cloudQueue);
}
}
}
else
{
// Not a valid event item - delete from queue and log (tell someone to fixe the blob event filter)
QueueService.DeleteQueueItem(queueName, queueItem, cloudQueue);
}
System.Threading.Thread.Sleep(500); // Just so the UI does not scroll too fast
} // Process Item
} // DeleteQueueItem
public static void IncreaseQueueItemLock(string queueName, Microsoft.WindowsAzure.Storage.Queue.CloudQueueMessage cloudQueueMessage, TimeSpan visibilityTimeout, Microsoft.WindowsAzure.Storage.Queue.CloudQueue cloudQueue)
{
string containerName = queueName.ToLower(); // must be lower case!
cloudQueue.UpdateMessageAsync(cloudQueueMessage, visibilityTimeout, Microsoft.WindowsAzure.Storage.Queue.MessageUpdateFields.Visibility);
} // DeleteQueueItem
/// <summary>
/// Returns the 1st message as a queue item
/// </summary>
/// <param name="queueName"></param>
/// <returns></returns>
public static Microsoft.WindowsAzure.Storage.Queue.CloudQueueMessage GetQueueItem(string queueName, Microsoft.WindowsAzure.Storage.Queue.CloudQueue cloudQueue)
{
string containerName = queueName.ToLower(); // must be lower case!
Microsoft.WindowsAzure.Storage.Queue.CloudQueueMessage cloudQueueMessage = cloudQueue.GetMessagesAsync(1).Result.FirstOrDefault();
return(cloudQueueMessage);
} // GetQueueItem
