Exemplo n.º 1
0
        public static bool WriteFileDataToInventoryDataTable(FileInventoryEntity fi)
        {
            try
            {
                EventSourceWriter.Log.MessageMethod(
                    $"Adding Message to Queue for File: {fi.RowKey}, Container: {fi.PartitionKey}");

                var account     = Microsoft.WindowsAzure.Storage.CloudStorageAccount.Parse(CloudConfigurationManager.GetSetting("StorageConnectionString"));
                var tableClient = account.CreateCloudTableClient();
                var table       = tableClient.GetTableReference(CloudConfigurationManager.GetSetting("FileInventory"));
                table.CreateIfNotExists();

                try
                {
                    // Create an operation to add the new customer to the people table.
                    var insertFileEntry = TableOperation.Insert(fi);

                    // Submit the operation to the table service.
                    table.Execute(insertFileEntry);
                    return(true);
                }
                catch
                {
                    return(false);
                }
            }
            catch (Exception ex)
            {
                EventSourceWriter.Log.MessageMethod(
                    $"Exception in Storage Monitor WriteFileDataToInventoryDataTable{ex.Message}");
                return(false);
            }
        }
Exemplo n.º 2
0
        public static void AddmessagetoQueue(FileInventoryEntity entity, string strConnectionString, string strType)
        {
            try
            {
                EventSourceWriter.Log.MessageMethod($"Adding Message to Queue for file {entity.RowKey}");

                // Retrieve storage account from connection string
                var account = Microsoft.WindowsAzure.Storage.CloudStorageAccount.Parse(CloudConfigurationManager.GetSetting("StorageConnectionString"));

                // Create the queue client
                var queueClient = account.CreateCloudQueueClient();

                // Retrieve a reference to a queue
                var queue = queueClient.GetQueueReference("urbanwaterqueue");

                // Create the queue if it doesn't already exist
                queue.CreateIfNotExists();

                // Create the queue message.
                //Meesage must be in the order type, filename, container, size, connection string
                var queueMessageString =
                    entity.RowKey + "," +
                    entity.PartitionKey + "," +
                    entity.LngFileLength + "," +
                    strConnectionString;

                var queueMessage = new Microsoft.WindowsAzure.Storage.Queue.CloudQueueMessage(queueMessageString);
                queue.AddMessage(queueMessage);
            }
            catch (Exception ex)
            {
                EventSourceWriter.Log.MessageMethod("Exception in AddmessagetoQueue" + ex.Message);
            }
        }
Exemplo n.º 3
0
        public void MonitorBlobStorage(string strType, string strConnectionString, string strTableName, List <string> containers)
        {
            try
            {
                // Get account details
                var account = Microsoft.WindowsAzure.Storage.CloudStorageAccount.Parse(CloudConfigurationManager.GetSetting(strConnectionString));

                // Create the blob and table client.
                CloudBlobClient  blobClient  = account.CreateCloudBlobClient();
                CloudTableClient tableClient = account.CreateCloudTableClient();

                // Create the table if it doesn't exist
                CloudTable table = tableClient.GetTableReference(strTableName);
                table.CreateIfNotExists();

                // Loop through each container, usually one for each utility
                foreach (string strContainer in containers)
                {
                    // Retrieve reference to a previously created container.
                    CloudBlobContainer container = blobClient.GetContainerReference(strContainer);

                    // Loop over items (files) within the container and output the length and URI.
                    foreach (IListBlobItem item in container.ListBlobs())
                    {
                        if (item.GetType() == typeof(CloudBlockBlob))
                        {
                            CloudBlockBlob blob = (CloudBlockBlob)item;

                            string strName = blob.Uri.ToString();
                            strName = Path.GetFileName(strName);

                            // Retrieve the entity with partition key of "Smith" and row key of "Jeff"
                            TableOperation fileNameQuery = TableOperation.Retrieve <FileInventoryEntity>(strContainer, strName);

                            // Retrieve entity
                            FileInventoryEntity fileNameEntity = (FileInventoryEntity)table.Execute(fileNameQuery).Result;

                            // If the current file doesn't exist, add an entry
                            if (fileNameEntity == null)
                            {
                                //if a new file, make entity and add to table
                                FileInventoryEntity inventoryEntity = new FileInventoryEntity();
                                inventoryEntity.PartitionKey   = strContainer;
                                inventoryEntity.RowKey         = strName;
                                inventoryEntity.lngFileLength  = blob.Properties.Length;
                                inventoryEntity.Etag           = blob.Properties.ETag;
                                inventoryEntity.UploadDateTime = DateTime.Now;

                                bool test = StorageMonitorUtility.WriteFileDataToInventoryDataTable(inventoryEntity, strConnectionString, strTableName);

                                //call recursive etag check on file to check is it uploaded
                                StorageMonitorUtility.CheckETagOfAddedFile(inventoryEntity, strConnectionString, strType);
                            }
                        }
                    }
                }
            }
            catch (Exception ex)
            {
                EventSourceWriter.Log.MessageMethod("Exception in storage moitoring MonitorBlobStorage" + ex.Message.ToString());
            }
        }
Exemplo n.º 4
0
        public static void CheckETagOfAddedFile(FileInventoryEntity entity)
        {
            try
            {
                EventSourceWriter.Log.MessageMethod($"Checking Etag of added file {entity.RowKey}");
                var blCondition = false;
                var etag        = entity.Etag;

                var account = Microsoft.WindowsAzure.Storage.CloudStorageAccount.Parse(CloudConfigurationManager.GetSetting("StorageConnectionString"));

                // Create the blob client.
                var blobClient = account.CreateCloudBlobClient();

                // Retrieve reference to a previously created container.
                var container = blobClient.GetContainerReference(entity.PartitionKey);

                do
                {
                    var blobs = container.ListBlobs();



                    if (blobs.Count() > 0 && blobs.FirstOrDefault().GetType() == typeof(CloudBlobDirectory))
                    {
                        foreach (CloudBlobDirectory outerdirectory in blobs)
                        {
                            var outerDirectoryBlobs = outerdirectory.ListBlobs();

                            while (outerDirectoryBlobs.Any() && outerDirectoryBlobs.FirstOrDefault().GetType() == typeof(CloudBlobDirectory))
                            {
                                var directory = (CloudBlobDirectory)outerDirectoryBlobs.FirstOrDefault();
                                outerDirectoryBlobs = directory.ListBlobs();
                            }

                            foreach (IListBlobItem item in outerDirectoryBlobs)
                            {
                                if (item is CloudBlockBlob)
                                {
                                    var blob = (CloudBlockBlob)item;

                                    var strName = blob.Uri.ToString();
                                    strName = Path.GetFileName(strName);

                                    if (strName == entity.RowKey)
                                    {
                                        if (blob.Properties.ETag == etag)
                                        {
                                            AddmessagetoQueue(entity, "StorageConnectionString", entity.PartitionKey);
                                            blCondition = true;
                                            break;
                                        }

                                        else
                                        {
                                            etag = entity.Etag;
                                            Thread.Sleep(1000);
                                        }
                                    }
                                }
                            }
                        }
                    }
                    else if (blobs.Count() > 0)
                    {
                        foreach (IListBlobItem item in blobs)
                        {
                            if (item is CloudBlockBlob)
                            {
                                var blob = (CloudBlockBlob)item;

                                var strName = blob.Uri.ToString();
                                strName = Path.GetFileName(strName);

                                if (strName == entity.RowKey)
                                {
                                    if (blob.Properties.ETag == etag)
                                    {
                                        AddmessagetoQueue(entity, "StorageConnectionString", entity.PartitionKey);
                                        blCondition = true;
                                        break;
                                    }

                                    else
                                    {
                                        etag = entity.Etag;
                                        Thread.Sleep(1000);
                                    }
                                }
                            }
                        }
                    }
                } while (!blCondition);
            }
            catch (Exception ex)
            {
                EventSourceWriter.Log.MessageMethod($"Exception in CheckETagOfAddedFile {ex.Message}");
            }
        }
Exemplo n.º 5
0
        public void MonitorBlobStorage()
        {
            try
            {
                // Get account details
                var account = Microsoft.WindowsAzure.Storage.CloudStorageAccount.Parse(CloudConfigurationManager.GetSetting("StorageConnectionString"));

                // Create the blob and table client.
                var blobClient  = account.CreateCloudBlobClient();
                var tableClient = account.CreateCloudTableClient();

                // Create the file logging table if it doesn't exist
                var table = tableClient.GetTableReference(CloudConfigurationManager.GetSetting("FileInventory"));
                table.CreateIfNotExists();

                // Get a list of all the active containers
                var currentContext = new SQLAzureDataContext();
                var containers     = currentContext.Containers.Where(x => x.Active).Select(x => x.FileContainers).ToList();

                // Loop through each container, usually one for each utility
                foreach (var strContainer in containers)
                {
                    // Retrieve reference to a previously created container.
                    var container = blobClient.GetContainerReference(strContainer);
                    container.CreateIfNotExists();

                    var blobs = container.ListBlobs();

                    if (blobs.Count() > 0 && blobs.FirstOrDefault().GetType() == typeof(CloudBlobDirectory))
                    {
                        foreach (CloudBlobDirectory outerdirectory in blobs)
                        {
                            var outerDirectoryBlobs = outerdirectory.ListBlobs();

                            while (outerDirectoryBlobs.Any() && outerDirectoryBlobs.FirstOrDefault().GetType() == typeof(CloudBlobDirectory))
                            {
                                var directory = (CloudBlobDirectory)outerDirectoryBlobs.FirstOrDefault();
                                outerDirectoryBlobs = directory.ListBlobs();
                            }

                            // Loop over items (files) within the container and output the length and URI.
                            foreach (var item in outerDirectoryBlobs)
                            {
                                if (item is CloudBlockBlob)
                                {
                                    var blob = (CloudBlockBlob)item;

                                    var strName = blob.Uri.ToString();
                                    strName = Path.GetFileName(strName);

                                    var fileNameQuery = TableOperation.Retrieve <FileInventoryEntity>(strContainer, strName);

                                    // Retrieve entity
                                    var fileNameEntity = (FileInventoryEntity)table.Execute(fileNameQuery).Result;


                                    // If the current file doesn't exist, add an entry
                                    if (fileNameEntity == null)
                                    {
                                        //if a new file, make entity and add to table
                                        var inventoryEntity = new FileInventoryEntity
                                        {
                                            PartitionKey   = strContainer,
                                            RowKey         = strName,
                                            LngFileLength  = blob.Properties.Length,
                                            Etag           = blob.Properties.ETag,
                                            UploadDateTime = DateTime.Now
                                        };

                                        StorageMonitorUtility.WriteFileDataToInventoryDataTable(inventoryEntity);

                                        //call recursive etag check on file to check is it uploaded
                                        StorageMonitorUtility.CheckETagOfAddedFile(inventoryEntity);
                                    }
                                }
                            }
                        }
                    }
                    else
                    {
                        foreach (var item in blobs)
                        {
                            if (item is CloudBlockBlob)
                            {
                                var blob = (CloudBlockBlob)item;

                                var strName = blob.Uri.ToString();
                                strName = Path.GetFileName(strName);

                                var fileNameQuery = TableOperation.Retrieve <FileInventoryEntity>(strContainer, strName);

                                // Retrieve entity
                                var fileNameEntity = (FileInventoryEntity)table.Execute(fileNameQuery).Result;


                                // If the current file doesn't exist, add an entry
                                if (fileNameEntity == null)
                                {
                                    //if a new file, make entity and add to table
                                    var inventoryEntity = new FileInventoryEntity
                                    {
                                        PartitionKey   = strContainer,
                                        RowKey         = strName,
                                        LngFileLength  = blob.Properties.Length,
                                        Etag           = blob.Properties.ETag,
                                        UploadDateTime = DateTime.Now
                                    };

                                    StorageMonitorUtility.WriteFileDataToInventoryDataTable(inventoryEntity);

                                    //call recursive etag check on file to check is it uploaded
                                    StorageMonitorUtility.CheckETagOfAddedFile(inventoryEntity);
                                }
                            }
                        }
                    }
                }
            }
            catch (Exception ex)
            {
                EventSourceWriter.Log.MessageMethod($"Exception in storage moitoring MonitorBlobStorage{ex.Message}");
            }
        }