public static bool WriteFileDataToInventoryDataTable(FileInventoryEntity fi) { try { EventSourceWriter.Log.MessageMethod( $"Adding Message to Queue for File: {fi.RowKey}, Container: {fi.PartitionKey}"); var account = Microsoft.WindowsAzure.Storage.CloudStorageAccount.Parse(CloudConfigurationManager.GetSetting("StorageConnectionString")); var tableClient = account.CreateCloudTableClient(); var table = tableClient.GetTableReference(CloudConfigurationManager.GetSetting("FileInventory")); table.CreateIfNotExists(); try { // Create an operation to add the new customer to the people table. var insertFileEntry = TableOperation.Insert(fi); // Submit the operation to the table service. table.Execute(insertFileEntry); return(true); } catch { return(false); } } catch (Exception ex) { EventSourceWriter.Log.MessageMethod( $"Exception in Storage Monitor WriteFileDataToInventoryDataTable{ex.Message}"); return(false); } }
public static void AddmessagetoQueue(FileInventoryEntity entity, string strConnectionString, string strType) { try { EventSourceWriter.Log.MessageMethod($"Adding Message to Queue for file {entity.RowKey}"); // Retrieve storage account from connection string var account = Microsoft.WindowsAzure.Storage.CloudStorageAccount.Parse(CloudConfigurationManager.GetSetting("StorageConnectionString")); // Create the queue client var queueClient = account.CreateCloudQueueClient(); // Retrieve a reference to a queue var queue = queueClient.GetQueueReference("urbanwaterqueue"); // Create the queue if it doesn't already exist queue.CreateIfNotExists(); // Create the queue message. //Meesage must be in the order type, filename, container, size, connection string var queueMessageString = entity.RowKey + "," + entity.PartitionKey + "," + entity.LngFileLength + "," + strConnectionString; var queueMessage = new Microsoft.WindowsAzure.Storage.Queue.CloudQueueMessage(queueMessageString); queue.AddMessage(queueMessage); } catch (Exception ex) { EventSourceWriter.Log.MessageMethod("Exception in AddmessagetoQueue" + ex.Message); } }
public void MonitorBlobStorage(string strType, string strConnectionString, string strTableName, List <string> containers) { try { // Get account details var account = Microsoft.WindowsAzure.Storage.CloudStorageAccount.Parse(CloudConfigurationManager.GetSetting(strConnectionString)); // Create the blob and table client. CloudBlobClient blobClient = account.CreateCloudBlobClient(); CloudTableClient tableClient = account.CreateCloudTableClient(); // Create the table if it doesn't exist CloudTable table = tableClient.GetTableReference(strTableName); table.CreateIfNotExists(); // Loop through each container, usually one for each utility foreach (string strContainer in containers) { // Retrieve reference to a previously created container. CloudBlobContainer container = blobClient.GetContainerReference(strContainer); // Loop over items (files) within the container and output the length and URI. foreach (IListBlobItem item in container.ListBlobs()) { if (item.GetType() == typeof(CloudBlockBlob)) { CloudBlockBlob blob = (CloudBlockBlob)item; string strName = blob.Uri.ToString(); strName = Path.GetFileName(strName); // Retrieve the entity with partition key of "Smith" and row key of "Jeff" TableOperation fileNameQuery = TableOperation.Retrieve <FileInventoryEntity>(strContainer, strName); // Retrieve entity FileInventoryEntity fileNameEntity = (FileInventoryEntity)table.Execute(fileNameQuery).Result; // If the current file doesn't exist, add an entry if (fileNameEntity == null) { //if a new file, make entity and add to table FileInventoryEntity inventoryEntity = new FileInventoryEntity(); inventoryEntity.PartitionKey = strContainer; inventoryEntity.RowKey = strName; inventoryEntity.lngFileLength = blob.Properties.Length; inventoryEntity.Etag = blob.Properties.ETag; inventoryEntity.UploadDateTime = DateTime.Now; bool test = StorageMonitorUtility.WriteFileDataToInventoryDataTable(inventoryEntity, strConnectionString, strTableName); //call recursive etag check on file to check is it uploaded StorageMonitorUtility.CheckETagOfAddedFile(inventoryEntity, strConnectionString, strType); } } } } } catch (Exception ex) { EventSourceWriter.Log.MessageMethod("Exception in storage moitoring MonitorBlobStorage" + ex.Message.ToString()); } }
public static void CheckETagOfAddedFile(FileInventoryEntity entity) { try { EventSourceWriter.Log.MessageMethod($"Checking Etag of added file {entity.RowKey}"); var blCondition = false; var etag = entity.Etag; var account = Microsoft.WindowsAzure.Storage.CloudStorageAccount.Parse(CloudConfigurationManager.GetSetting("StorageConnectionString")); // Create the blob client. var blobClient = account.CreateCloudBlobClient(); // Retrieve reference to a previously created container. var container = blobClient.GetContainerReference(entity.PartitionKey); do { var blobs = container.ListBlobs(); if (blobs.Count() > 0 && blobs.FirstOrDefault().GetType() == typeof(CloudBlobDirectory)) { foreach (CloudBlobDirectory outerdirectory in blobs) { var outerDirectoryBlobs = outerdirectory.ListBlobs(); while (outerDirectoryBlobs.Any() && outerDirectoryBlobs.FirstOrDefault().GetType() == typeof(CloudBlobDirectory)) { var directory = (CloudBlobDirectory)outerDirectoryBlobs.FirstOrDefault(); outerDirectoryBlobs = directory.ListBlobs(); } foreach (IListBlobItem item in outerDirectoryBlobs) { if (item is CloudBlockBlob) { var blob = (CloudBlockBlob)item; var strName = blob.Uri.ToString(); strName = Path.GetFileName(strName); if (strName == entity.RowKey) { if (blob.Properties.ETag == etag) { AddmessagetoQueue(entity, "StorageConnectionString", entity.PartitionKey); blCondition = true; break; } else { etag = entity.Etag; Thread.Sleep(1000); } } } } } } else if (blobs.Count() > 0) { foreach (IListBlobItem item in blobs) { if (item is CloudBlockBlob) { var blob = (CloudBlockBlob)item; var strName = blob.Uri.ToString(); strName = Path.GetFileName(strName); if (strName == entity.RowKey) { if (blob.Properties.ETag == etag) { AddmessagetoQueue(entity, "StorageConnectionString", entity.PartitionKey); blCondition = true; break; } else { etag = entity.Etag; Thread.Sleep(1000); } } } } } } while (!blCondition); } catch (Exception ex) { EventSourceWriter.Log.MessageMethod($"Exception in CheckETagOfAddedFile {ex.Message}"); } }
public void MonitorBlobStorage() { try { // Get account details var account = Microsoft.WindowsAzure.Storage.CloudStorageAccount.Parse(CloudConfigurationManager.GetSetting("StorageConnectionString")); // Create the blob and table client. var blobClient = account.CreateCloudBlobClient(); var tableClient = account.CreateCloudTableClient(); // Create the file logging table if it doesn't exist var table = tableClient.GetTableReference(CloudConfigurationManager.GetSetting("FileInventory")); table.CreateIfNotExists(); // Get a list of all the active containers var currentContext = new SQLAzureDataContext(); var containers = currentContext.Containers.Where(x => x.Active).Select(x => x.FileContainers).ToList(); // Loop through each container, usually one for each utility foreach (var strContainer in containers) { // Retrieve reference to a previously created container. var container = blobClient.GetContainerReference(strContainer); container.CreateIfNotExists(); var blobs = container.ListBlobs(); if (blobs.Count() > 0 && blobs.FirstOrDefault().GetType() == typeof(CloudBlobDirectory)) { foreach (CloudBlobDirectory outerdirectory in blobs) { var outerDirectoryBlobs = outerdirectory.ListBlobs(); while (outerDirectoryBlobs.Any() && outerDirectoryBlobs.FirstOrDefault().GetType() == typeof(CloudBlobDirectory)) { var directory = (CloudBlobDirectory)outerDirectoryBlobs.FirstOrDefault(); outerDirectoryBlobs = directory.ListBlobs(); } // Loop over items (files) within the container and output the length and URI. foreach (var item in outerDirectoryBlobs) { if (item is CloudBlockBlob) { var blob = (CloudBlockBlob)item; var strName = blob.Uri.ToString(); strName = Path.GetFileName(strName); var fileNameQuery = TableOperation.Retrieve <FileInventoryEntity>(strContainer, strName); // Retrieve entity var fileNameEntity = (FileInventoryEntity)table.Execute(fileNameQuery).Result; // If the current file doesn't exist, add an entry if (fileNameEntity == null) { //if a new file, make entity and add to table var inventoryEntity = new FileInventoryEntity { PartitionKey = strContainer, RowKey = strName, LngFileLength = blob.Properties.Length, Etag = blob.Properties.ETag, UploadDateTime = DateTime.Now }; StorageMonitorUtility.WriteFileDataToInventoryDataTable(inventoryEntity); //call recursive etag check on file to check is it uploaded StorageMonitorUtility.CheckETagOfAddedFile(inventoryEntity); } } } } } else { foreach (var item in blobs) { if (item is CloudBlockBlob) { var blob = (CloudBlockBlob)item; var strName = blob.Uri.ToString(); strName = Path.GetFileName(strName); var fileNameQuery = TableOperation.Retrieve <FileInventoryEntity>(strContainer, strName); // Retrieve entity var fileNameEntity = (FileInventoryEntity)table.Execute(fileNameQuery).Result; // If the current file doesn't exist, add an entry if (fileNameEntity == null) { //if a new file, make entity and add to table var inventoryEntity = new FileInventoryEntity { PartitionKey = strContainer, RowKey = strName, LngFileLength = blob.Properties.Length, Etag = blob.Properties.ETag, UploadDateTime = DateTime.Now }; StorageMonitorUtility.WriteFileDataToInventoryDataTable(inventoryEntity); //call recursive etag check on file to check is it uploaded StorageMonitorUtility.CheckETagOfAddedFile(inventoryEntity); } } } } } } catch (Exception ex) { EventSourceWriter.Log.MessageMethod($"Exception in storage moitoring MonitorBlobStorage{ex.Message}"); } }