public static bool WriteFileDataToInventoryDataTable(FileInventoryEntity fi)
{
try
{
EventSourceWriter.Log.MessageMethod(
$"Adding Message to Queue for File: {fi.RowKey}, Container: {fi.PartitionKey}");
var account = Microsoft.WindowsAzure.Storage.CloudStorageAccount.Parse(CloudConfigurationManager.GetSetting("StorageConnectionString"));
var tableClient = account.CreateCloudTableClient();
var table = tableClient.GetTableReference(CloudConfigurationManager.GetSetting("FileInventory"));
table.CreateIfNotExists();
try
{
// Create an operation to add the new customer to the people table.
var insertFileEntry = TableOperation.Insert(fi);
// Submit the operation to the table service.
table.Execute(insertFileEntry);
return(true);
}
catch
{
return(false);
}
}
catch (Exception ex)
{
EventSourceWriter.Log.MessageMethod(
$"Exception in Storage Monitor WriteFileDataToInventoryDataTable{ex.Message}");
return(false);
}
}
public static void AddmessagetoQueue(FileInventoryEntity entity, string strConnectionString, string strType)
{
try
{
EventSourceWriter.Log.MessageMethod($"Adding Message to Queue for file {entity.RowKey}");
// Retrieve storage account from connection string
var account = Microsoft.WindowsAzure.Storage.CloudStorageAccount.Parse(CloudConfigurationManager.GetSetting("StorageConnectionString"));
// Create the queue client
var queueClient = account.CreateCloudQueueClient();
// Retrieve a reference to a queue
var queue = queueClient.GetQueueReference("urbanwaterqueue");
// Create the queue if it doesn't already exist
queue.CreateIfNotExists();
// Create the queue message.
//Meesage must be in the order type, filename, container, size, connection string
var queueMessageString =
entity.RowKey + "," +
entity.PartitionKey + "," +
entity.LngFileLength + "," +
strConnectionString;
var queueMessage = new Microsoft.WindowsAzure.Storage.Queue.CloudQueueMessage(queueMessageString);
queue.AddMessage(queueMessage);
}
catch (Exception ex)
{
EventSourceWriter.Log.MessageMethod("Exception in AddmessagetoQueue" + ex.Message);
}
}
public void MonitorBlobStorage(string strType, string strConnectionString, string strTableName, List <string> containers)
{
try
{
// Get account details
var account = Microsoft.WindowsAzure.Storage.CloudStorageAccount.Parse(CloudConfigurationManager.GetSetting(strConnectionString));
// Create the blob and table client.
CloudBlobClient blobClient = account.CreateCloudBlobClient();
CloudTableClient tableClient = account.CreateCloudTableClient();
// Create the table if it doesn't exist
CloudTable table = tableClient.GetTableReference(strTableName);
table.CreateIfNotExists();
// Loop through each container, usually one for each utility
foreach (string strContainer in containers)
{
// Retrieve reference to a previously created container.
CloudBlobContainer container = blobClient.GetContainerReference(strContainer);
// Loop over items (files) within the container and output the length and URI.
foreach (IListBlobItem item in container.ListBlobs())
{
if (item.GetType() == typeof(CloudBlockBlob))
{
CloudBlockBlob blob = (CloudBlockBlob)item;
string strName = blob.Uri.ToString();
strName = Path.GetFileName(strName);
// Retrieve the entity with partition key of "Smith" and row key of "Jeff"
TableOperation fileNameQuery = TableOperation.Retrieve <FileInventoryEntity>(strContainer, strName);
// Retrieve entity
FileInventoryEntity fileNameEntity = (FileInventoryEntity)table.Execute(fileNameQuery).Result;
// If the current file doesn't exist, add an entry
if (fileNameEntity == null)
{
//if a new file, make entity and add to table
FileInventoryEntity inventoryEntity = new FileInventoryEntity();
inventoryEntity.PartitionKey = strContainer;
inventoryEntity.RowKey = strName;
inventoryEntity.lngFileLength = blob.Properties.Length;
inventoryEntity.Etag = blob.Properties.ETag;
inventoryEntity.UploadDateTime = DateTime.Now;
bool test = StorageMonitorUtility.WriteFileDataToInventoryDataTable(inventoryEntity, strConnectionString, strTableName);
//call recursive etag check on file to check is it uploaded
StorageMonitorUtility.CheckETagOfAddedFile(inventoryEntity, strConnectionString, strType);
}
}
}
}
}
catch (Exception ex)
{
EventSourceWriter.Log.MessageMethod("Exception in storage moitoring MonitorBlobStorage" + ex.Message.ToString());
}
}
public static void CheckETagOfAddedFile(FileInventoryEntity entity)
{
try
{
EventSourceWriter.Log.MessageMethod($"Checking Etag of added file {entity.RowKey}");
var blCondition = false;
var etag = entity.Etag;
var account = Microsoft.WindowsAzure.Storage.CloudStorageAccount.Parse(CloudConfigurationManager.GetSetting("StorageConnectionString"));
// Create the blob client.
var blobClient = account.CreateCloudBlobClient();
// Retrieve reference to a previously created container.
var container = blobClient.GetContainerReference(entity.PartitionKey);
do
{
var blobs = container.ListBlobs();
if (blobs.Count() > 0 && blobs.FirstOrDefault().GetType() == typeof(CloudBlobDirectory))
{
foreach (CloudBlobDirectory outerdirectory in blobs)
{
var outerDirectoryBlobs = outerdirectory.ListBlobs();
while (outerDirectoryBlobs.Any() && outerDirectoryBlobs.FirstOrDefault().GetType() == typeof(CloudBlobDirectory))
{
var directory = (CloudBlobDirectory)outerDirectoryBlobs.FirstOrDefault();
outerDirectoryBlobs = directory.ListBlobs();
}
foreach (IListBlobItem item in outerDirectoryBlobs)
{
if (item is CloudBlockBlob)
{
var blob = (CloudBlockBlob)item;
var strName = blob.Uri.ToString();
strName = Path.GetFileName(strName);
if (strName == entity.RowKey)
{
if (blob.Properties.ETag == etag)
{
AddmessagetoQueue(entity, "StorageConnectionString", entity.PartitionKey);
blCondition = true;
break;
}
else
{
etag = entity.Etag;
Thread.Sleep(1000);
}
}
}
}
}
}
else if (blobs.Count() > 0)
{
foreach (IListBlobItem item in blobs)
{
if (item is CloudBlockBlob)
{
var blob = (CloudBlockBlob)item;
var strName = blob.Uri.ToString();
strName = Path.GetFileName(strName);
if (strName == entity.RowKey)
{
if (blob.Properties.ETag == etag)
{
AddmessagetoQueue(entity, "StorageConnectionString", entity.PartitionKey);
blCondition = true;
break;
}
else
{
etag = entity.Etag;
Thread.Sleep(1000);
}
}
}
}
}
} while (!blCondition);
}
catch (Exception ex)
{
EventSourceWriter.Log.MessageMethod($"Exception in CheckETagOfAddedFile {ex.Message}");
}
}
public void MonitorBlobStorage()
{
try
{
// Get account details
var account = Microsoft.WindowsAzure.Storage.CloudStorageAccount.Parse(CloudConfigurationManager.GetSetting("StorageConnectionString"));
// Create the blob and table client.
var blobClient = account.CreateCloudBlobClient();
var tableClient = account.CreateCloudTableClient();
// Create the file logging table if it doesn't exist
var table = tableClient.GetTableReference(CloudConfigurationManager.GetSetting("FileInventory"));
table.CreateIfNotExists();
// Get a list of all the active containers
var currentContext = new SQLAzureDataContext();
var containers = currentContext.Containers.Where(x => x.Active).Select(x => x.FileContainers).ToList();
// Loop through each container, usually one for each utility
foreach (var strContainer in containers)
{
// Retrieve reference to a previously created container.
var container = blobClient.GetContainerReference(strContainer);
container.CreateIfNotExists();
var blobs = container.ListBlobs();
if (blobs.Count() > 0 && blobs.FirstOrDefault().GetType() == typeof(CloudBlobDirectory))
{
foreach (CloudBlobDirectory outerdirectory in blobs)
{
var outerDirectoryBlobs = outerdirectory.ListBlobs();
while (outerDirectoryBlobs.Any() && outerDirectoryBlobs.FirstOrDefault().GetType() == typeof(CloudBlobDirectory))
{
var directory = (CloudBlobDirectory)outerDirectoryBlobs.FirstOrDefault();
outerDirectoryBlobs = directory.ListBlobs();
}
// Loop over items (files) within the container and output the length and URI.
foreach (var item in outerDirectoryBlobs)
{
if (item is CloudBlockBlob)
{
var blob = (CloudBlockBlob)item;
var strName = blob.Uri.ToString();
strName = Path.GetFileName(strName);
var fileNameQuery = TableOperation.Retrieve <FileInventoryEntity>(strContainer, strName);
// Retrieve entity
var fileNameEntity = (FileInventoryEntity)table.Execute(fileNameQuery).Result;
// If the current file doesn't exist, add an entry
if (fileNameEntity == null)
{
//if a new file, make entity and add to table
var inventoryEntity = new FileInventoryEntity
{
PartitionKey = strContainer,
RowKey = strName,
LngFileLength = blob.Properties.Length,
Etag = blob.Properties.ETag,
UploadDateTime = DateTime.Now
};
StorageMonitorUtility.WriteFileDataToInventoryDataTable(inventoryEntity);
//call recursive etag check on file to check is it uploaded
StorageMonitorUtility.CheckETagOfAddedFile(inventoryEntity);
}
}
}
}
}
else
{
foreach (var item in blobs)
{
if (item is CloudBlockBlob)
{
var blob = (CloudBlockBlob)item;
var strName = blob.Uri.ToString();
strName = Path.GetFileName(strName);
var fileNameQuery = TableOperation.Retrieve <FileInventoryEntity>(strContainer, strName);
// Retrieve entity
var fileNameEntity = (FileInventoryEntity)table.Execute(fileNameQuery).Result;
// If the current file doesn't exist, add an entry
if (fileNameEntity == null)
{
//if a new file, make entity and add to table
var inventoryEntity = new FileInventoryEntity
{
PartitionKey = strContainer,
RowKey = strName,
LngFileLength = blob.Properties.Length,
Etag = blob.Properties.ETag,
UploadDateTime = DateTime.Now
};
StorageMonitorUtility.WriteFileDataToInventoryDataTable(inventoryEntity);
//call recursive etag check on file to check is it uploaded
StorageMonitorUtility.CheckETagOfAddedFile(inventoryEntity);
}
}
}
}
}
}
catch (Exception ex)
{
EventSourceWriter.Log.MessageMethod($"Exception in storage moitoring MonitorBlobStorage{ex.Message}");
}
}
