public static void Run([CosmosDBTrigger(
databaseName: "IoT",
collectionName: "IoT",
ConnectionStringSetting = "DBConnection",
LeaseCollectionName = "leases")] IReadOnlyList <Document> documents, TraceWriter log)
{
Microsoft.WindowsAzure.Storage.Queue.CloudQueueClient queueClient;
Microsoft.WindowsAzure.Storage.Queue.CloudQueue queue;
Microsoft.WindowsAzure.Storage.CloudStorageAccount storageAccount = Microsoft.WindowsAzure.Storage.CloudStorageAccount.Parse("DefaultEndpointsProtocol=https;AccountName=scmrstorage;AccountKey=FtW5Qlz/5rWiqX0MPlGO0X2anGs5t7ea/H/ZkdcIEHlTA9isEinpscnuuhw8GwKR+7+Eo2IDRG1jwdMoDsRTqg==;EndpointSuffix=core.windows.net");
queueClient = storageAccount.CreateCloudQueueClient();
queue = queueClient.GetQueueReference("trafficqueue");
foreach (var doc in documents)
{
if (doc.GetPropertyValue <string>("accident") == "true")
{
string m = String.Format("{{ \"lat\": {0}, \"long\": {1}, \"carId\": \"{2}\" }}",
doc.GetPropertyValue <string>("lat"),
doc.GetPropertyValue <string>("longitude"),
doc.GetPropertyValue <string>("carid"));
Microsoft.WindowsAzure.Storage.Queue.CloudQueueMessage message = new Microsoft.WindowsAzure.Storage.Queue.CloudQueueMessage
(m);
queue.AddMessage(message);
}
}
}
public static void Run([TimerTrigger("0 */5 * * * *")] TimerInfo myTimer, ILogger log)
{
try
{
log.LogInformation($"ProcessLandingDataUsingADF Timer trigger function executed at: {DateTime.Now}");
string cloudAccountName = null;
string cloudKey = null;
string queueName = "fileevent";
AzureServiceTokenProvider azureServiceTokenProvider = new AzureServiceTokenProvider();
KeyVaultClient keyvaultClient = new KeyVaultClient(new KeyVaultClient.AuthenticationCallback(azureServiceTokenProvider.KeyVaultTokenCallback));
string keyVault = Environment.GetEnvironmentVariable("MY_KEY_VAULT");
var secretCloudAccountName = keyvaultClient.GetSecretAsync($"https://{keyVault}.vault.azure.net/", "LandingZoneStorageAccountName");
var secretcloudKey = keyvaultClient.GetSecretAsync($"https://{keyVault}.vault.azure.net/", "LandingZoneStorageAccountKey");
cloudAccountName = secretCloudAccountName.Result.Value;
cloudKey = secretcloudKey.Result.Value;
Microsoft.WindowsAzure.Storage.Queue.CloudQueue cloudQueue = QueueService.ConnectToQueueStorage(queueName, cloudAccountName, cloudKey);
Microsoft.WindowsAzure.Storage.Queue.CloudQueueMessage queueItem = QueueService.GetQueueItem(queueName, cloudQueue);
while (queueItem != null)
{
ProcessItem(queueItem, queueName, cloudQueue, log);
queueItem = QueueService.GetQueueItem(queueName, cloudQueue);
}
}
catch (Exception ex)
{
log.LogError("ProcessLandingDataUsingADF Exception: " + ex.ToString());
}
} // Run
public static void Run([CosmosDBTrigger(
databaseName: "rafat-tweets",
collectionName: "tweets",
ConnectionStringSetting = "DBConnection",
LeaseCollectionName = "leases")] IReadOnlyList <Document> items, TraceWriter log)
{
if (_cognitiveSvc == null)
{
_cognitiveSvc = new CognitiveSvc("Ocp-Apim-Subscription-Key", "Key");
}
//Queue Conneciton
Microsoft.WindowsAzure.Storage.Queue.CloudQueueClient queueClient;
Microsoft.WindowsAzure.Storage.Queue.CloudQueue queue;
CloudStorageAccount storageAccount = CloudStorageAccount.Parse("DefaultEndpointsProtocol=https;AccountName=scmrstorage;AccountKey=AccountKey;EndpointSuffix=core.windows.net");
queueClient = storageAccount.CreateCloudQueueClient();
queue = queueClient.GetQueueReference("tweetqueue");
foreach (var doc in items)
{
string _post = doc.GetPropertyValue <string>("Post");
float _sentiments = _cognitiveSvc.GetSentiments(_post).Result;
string m = String.Format("{{ \"Author\": \"{0}\", \"Post\":\"{1}\", \"Sentiments\": {2} }}",
doc.GetPropertyValue <string>("Author"),
_post,
_sentiments);
Microsoft.WindowsAzure.Storage.Queue.CloudQueueMessage message =
new Microsoft.WindowsAzure.Storage.Queue.CloudQueueMessage(m);
queue.AddMessage(message);
log.Verbose("Message Added");
}
}
public static void AddmessagetoQueue(FileInventoryEntity entity, string strConnectionString, string strType)
{
try
{
EventSourceWriter.Log.MessageMethod($"Adding Message to Queue for file {entity.RowKey}");
// Retrieve storage account from connection string
var account = Microsoft.WindowsAzure.Storage.CloudStorageAccount.Parse(CloudConfigurationManager.GetSetting("StorageConnectionString"));
// Create the queue client
var queueClient = account.CreateCloudQueueClient();
// Retrieve a reference to a queue
var queue = queueClient.GetQueueReference("urbanwaterqueue");
// Create the queue if it doesn't already exist
queue.CreateIfNotExists();
// Create the queue message.
//Meesage must be in the order type, filename, container, size, connection string
var queueMessageString =
entity.RowKey + "," +
entity.PartitionKey + "," +
entity.LngFileLength + "," +
strConnectionString;
var queueMessage = new Microsoft.WindowsAzure.Storage.Queue.CloudQueueMessage(queueMessageString);
queue.AddMessage(queueMessage);
}
catch (Exception ex)
{
EventSourceWriter.Log.MessageMethod("Exception in AddmessagetoQueue" + ex.Message);
}
}
} // GetQueueItem
/// <summary>
/// Removes a message from the queue
/// </summary>
/// <param name="queueName"></param>
/// <param name="cloudQueueMessage"></param>
public static void DeleteQueueItem(string queueName, Microsoft.WindowsAzure.Storage.Queue.CloudQueueMessage cloudQueueMessage,
Microsoft.WindowsAzure.Storage.Queue.CloudQueue cloudQueue)
{
string containerName = queueName.ToLower(); // must be lower case!
cloudQueue.DeleteMessageAsync(cloudQueueMessage);
} // DeleteQueueItem
/// <summary>
/// Returns the 1st message as a queue item
/// </summary>
/// <param name="queueName"></param>
/// <returns></returns>
public static Microsoft.WindowsAzure.Storage.Queue.CloudQueueMessage GetQueueItem(string queueName, Microsoft.WindowsAzure.Storage.Queue.CloudQueue cloudQueue)
{
string containerName = queueName.ToLower(); // must be lower case!
Microsoft.WindowsAzure.Storage.Queue.CloudQueueMessage cloudQueueMessage = cloudQueue.GetMessagesAsync(1).Result.FirstOrDefault();
return(cloudQueueMessage);
} // GetQueueItem
/// <summary>
/// Removes a message from the queue
/// </summary>
/// <param name="queueContainer"></param>
/// <param name="cloudQueueMessage"></param>
public void DeleteQueueItem(string queueContainer, Microsoft.WindowsAzure.Storage.Queue.CloudQueueMessage cloudQueueMessage)
{
InitializeStorage(queueContainer);
string containerName = queueContainer.ToString().ToLower(); // must be lower case!
Microsoft.WindowsAzure.Storage.Queue.CloudQueueClient queueStorage = queueStorageDictionary[containerName];
Microsoft.WindowsAzure.Storage.Queue.CloudQueue cloudQueue = queueStorage.GetQueueReference(containerName);
cloudQueue.DeleteMessage(cloudQueueMessage);
} // DeleteQueueItem
} // DeleteQueueItem
/// <summary>
/// Returns the 1st message as a queue item
/// </summary>
/// <param name="queueContainer"></param>
/// <returns></returns>
public Microsoft.WindowsAzure.Storage.Queue.CloudQueueMessage GetQueueItem(string queueContainer)
{
InitializeStorage(queueContainer);
string containerName = queueContainer.ToString().ToLower(); // must be lower case!
Microsoft.WindowsAzure.Storage.Queue.CloudQueueClient queueStorage = queueStorageDictionary[containerName];
Microsoft.WindowsAzure.Storage.Queue.CloudQueue cloudQueue = queueStorage.GetQueueReference(containerName);
Microsoft.WindowsAzure.Storage.Queue.CloudQueueMessage cloudQueueMessage = cloudQueue.GetMessage(TimeSpan.FromMinutes(1));
return(cloudQueueMessage);
} // GetQueueItem
/// <summary>
/// Grabs an item from the queue locking it for a specified timespan (you need to call DeleteItem to remove
/// it from the queue)
/// </summary>
/// <param name="queueName"></param>
/// <param name="timeSpan"></param>
/// <returns></returns>
public Model.Queue.QueueModel Dequeue(string queueName, TimeSpan timeSpan)
{
AzureQueueHelper azureQueueHelper = new AzureQueueHelper();
DateTime expirationDate = DateTime.UtcNow.Add(timeSpan);
Microsoft.WindowsAzure.Storage.Queue.CloudQueueMessage item = azureQueueHelper.GetQueueItem(queueName.ToLower());
if (item == null)
{
return(null);
}
else
{
Model.Queue.QueueModel queueModel = new Model.Queue.QueueModel();
queueModel.Id = item.Id;
queueModel.PopReceipt = item.PopReceipt;
queueModel.Item = item.AsString;
queueModel.LockExpirationInUTC = expirationDate;
return(queueModel);
}
}
// POST api/bsm
public HttpResponseMessage Post([FromBody] BsmBundle bundle)
{
Trace.TraceInformation("[TRACE] Entering BsmController::Post(BsmBundle)...");
if (srBsmQueue == null)
{
Trace.TraceError("Unable to add BsmBundle to queue-- Inbound BSM queue not created");
Trace.TraceInformation("[TRACE] Exiting BsmController::Post(BsmBundle)...");
return(this.Request.CreateErrorResponse(HttpStatusCode.BadRequest, "Inbound BSM queue does not exist"));
}
if (this.ModelState.IsValid)
{
string strQueueMessage = Newtonsoft.Json.JsonConvert.SerializeObject(bundle);
Microsoft.WindowsAzure.Storage.Queue.CloudQueueMessage rMessage =
new Microsoft.WindowsAzure.Storage.Queue.CloudQueueMessage(strQueueMessage);
const int hours = 1;
const int mins = 0;
const int secs = 0;
TimeSpan rTimeToLive = new TimeSpan(hours, mins, secs);
Trace.TraceInformation("Adding BsmBundle message to inbound BSM queue...");
srBsmQueue.AddMessage(rMessage, rTimeToLive);
Trace.TraceInformation("[TRACE] Exiting BsmController::Post(BsmBundle)...");
return(this.Request.CreateResponse(HttpStatusCode.OK));
}
else
{
Trace.TraceError("Unable to add BsmBundle to queue-- BsmBundle message not properly formatted");
Trace.TraceInformation("[TRACE] Exiting BsmController::Post(BsmBundle)...");
return(this.Request.CreateErrorResponse(HttpStatusCode.BadRequest, "BsmBundle message not properly formatted"));
}
}
} // Run
static void ProcessItem(Microsoft.WindowsAzure.Storage.Queue.CloudQueueMessage queueItem, string queueName, Microsoft.WindowsAzure.Storage.Queue.CloudQueue cloudQueue, ILogger log)
{
log.LogInformation("ProcessItem:" + queueItem);
JSONService jsonService = new JSONService();
FileEvent fileEvent = jsonService.Deserialize <FileEvent>(queueItem.AsString);
log.LogInformation("ProcessItem Topic: " + fileEvent.topic);
log.LogInformation("ProcessItem Subject: " + fileEvent.subject);
// Make sure we have the correct event
if (fileEvent.eventType == "Microsoft.Storage.BlobCreated") // && fileEvent.subject.ToLower().EndsWith("end_file.txt"))
{
// e.g. "subject": "/blobServices/default/containers/acmeinc/blobs/inbox/2020-04-13/test_end_file.txt"
// 01234567890123456789012345678901234567890123456789 14 -> 30 =16
// e.g. "subject": "/blobServices/default/containers/acmeinc/blobs/test_end_file.txt"
// 01234567890123456789012345678901234567890123456789 13 -> 13 = 0
string removedPrefix = fileEvent.subject.ToLower().Replace("/blobservices/default/containers/", string.Empty);
string customerId = removedPrefix.Substring(0, removedPrefix.IndexOf("/"));
string partitionId = removedPrefix.Substring(0, removedPrefix.IndexOf("/")); // this just happens to be the same as the customer id in the this example
// Get the Cosmos DB data as to which pipeline belongs to this customer.
DocumentDBRepository <CosmosIngestionData> documentDBRepository = new DocumentDBRepository <CosmosIngestionData>(log);
var result = documentDBRepository.GetItems(o => o.CustomerId == customerId && o.PartitionId == partitionId).FirstOrDefault();
if (result == null)
{
// Someone forgot to create the CosmosDB document (whoops), send an email and log. Give the developer "1" hour to get things setup.
log.LogInformation("Pipeline does not exist. Setting queue item to retry in 1 hour.");
QueueService.IncreaseQueueItemLock(queueName, queueItem, TimeSpan.FromHours(1), cloudQueue);
}
else
{
if (result.isADFEnabled)
{
// We are good to try to start the ADF
log.LogInformation("Starting Pipeline: " + result.ADFPipelineName);
try
{
// Try to start the pipeline (we are "trying" since technically something could have just disabled it :) )
DateTime dtNow = DateTime.UtcNow; // you could make this EST or such....
string inputFileSystem = customerId;
int blobsIndex = removedPrefix.IndexOf("/blobs/") + 7;
int lastSlashIndex = removedPrefix.LastIndexOf("/");
string inputFileDirectory = removedPrefix.Substring(blobsIndex, lastSlashIndex - blobsIndex);
string outputFileSystem = Environment.GetEnvironmentVariable("LandingZoneDataLakeContainer");
string outputFileDirectory = string.Format("customer-landing-data/{0}/{1:yyyy}/{2:MM}/{3:dd}", customerId, dtNow, dtNow, dtNow);
DataFactoryService.StartPipeline(inputFileSystem, inputFileDirectory, outputFileSystem, outputFileDirectory, result.ADFPipelineName, result.ADFResourceGroup, result.ADFDataFactoryName, result.ADFSubscriptionId, log);
QueueService.DeleteQueueItem(queueName, queueItem, cloudQueue);
}
catch (Exception ex)
{
// Pipeline could not start (it might be in a "provisioning state", set the queue item to try again in "5" minutes
log.LogInformation("Error starting Pipeline (item not dequeued): " + result.ADFPipelineName + " Error: " + ex.ToString());
QueueService.IncreaseQueueItemLock(queueName, queueItem, TimeSpan.FromMinutes(5), cloudQueue);
}
}
else
{
// Pipeline is not enabled, set the queue item to try again in "5" minutes
log.LogInformation("Pipeline: " + result.ADFPipelineName + " is not enabled. Setting queue item to retry in 5 minutes.");
QueueService.IncreaseQueueItemLock(queueName, queueItem, TimeSpan.FromMinutes(5), cloudQueue);
}
}
}
else
{
// Not a valid event item - delete from queue and log (tell someone to fixe the blob event filter)
QueueService.DeleteQueueItem(queueName, queueItem, cloudQueue);
}
System.Threading.Thread.Sleep(500); // Just so the UI does not scroll too fast
} // Process Item
} // DeleteQueueItem
public static void IncreaseQueueItemLock(string queueName, Microsoft.WindowsAzure.Storage.Queue.CloudQueueMessage cloudQueueMessage, TimeSpan visibilityTimeout, Microsoft.WindowsAzure.Storage.Queue.CloudQueue cloudQueue)
{
string containerName = queueName.ToLower(); // must be lower case!
cloudQueue.UpdateMessageAsync(cloudQueueMessage, visibilityTimeout, Microsoft.WindowsAzure.Storage.Queue.MessageUpdateFields.Visibility);
} // DeleteQueueItem
