/// <summary>
/// CopyBlobFromBackupToRestore
/// - Copies storage blob from backup SA container to the respective source/restore SA container
/// </summary>
/// <returns></returns>
public async Task <string> CopyBlobFromBackupToRestore(DestinationBlobInfo backupBlob)
{
string destinationStorageAccountConnectionString =
Environment.GetEnvironmentVariable("SRC_STORAGE_ACCOUNT_CONN");
string sourceStorageAccountConnectionString =
Environment.GetEnvironmentVariable("RES_STORAGE_ACCOUNT_CONN");
// Retrieve the storage account from the connection string.
CloudStorageAccount sourceStorageAccount = CloudStorageAccount.Parse(sourceStorageAccountConnectionString);
CloudBlobClient sourceBlobClient = sourceStorageAccount.CreateCloudBlobClient();
// Retrieve the storage account from the connection string.
CloudStorageAccount destinationStorageAccount = CloudStorageAccount.Parse(destinationStorageAccountConnectionString);
CloudBlobClient destinationBlobClient = destinationStorageAccount.CreateCloudBlobClient();
CloudBlobContainer sourceContainer = sourceBlobClient.GetContainerReference(backupBlob.ContainerName);
bool sourceContainerExists = await sourceContainer.ExistsAsync();
if (sourceContainerExists)
{
CloudBlockBlob sourceBlockBlob = sourceContainer.GetBlockBlobReference(backupBlob.BlobName);
bool sourceBlobExists = await sourceBlockBlob.ExistsAsync();
if (sourceBlobExists)
{
CloudBlobContainer destinationContainer = destinationBlobClient.GetContainerReference(backupBlob.OrgContainerName);
await destinationContainer.CreateIfNotExistsAsync();
CloudBlockBlob destinationBlob = destinationContainer.GetBlockBlobReference(backupBlob.OrgBlobName);
_logger.LogInformation($"About to sync copy from Container: {sourceBlockBlob.Container.Name}, Blob: {sourceBlockBlob.Name}. Blob size: {sourceBlockBlob.Properties.Length} bytes");
// copyResult = "SYNCCOPY";
string copyResult = await destinationBlob.StartCopyAsync(sourceBlockBlob);
_logger.LogInformation($"Copy Scheduled. Source Blob Name: {backupBlob.BlobName}, Destination Blob Name: {backupBlob.OrgBlobName}, Copy Id: {copyResult}.");
return(copyResult);
}
else
{
_logger.LogInformation($"Not able to locate the blob: {backupBlob.BlobName} in source storage account.");
}
}
else
{
_logger.LogInformation($"Not able to locate the container: {backupBlob.ContainerName} in source storage account.");
}
return(string.Empty);
}
/// <summary>
/// CopyBlobFromSourceToBackup:
/// - Copies blob from source SA container to destination/backup SA container
/// </summary>
/// <returns></returns>
public async Task <DestinationBlobInfo> CopyBlobFromSourceToBackup(IBlobEvent eventData)
{
DestinationBlobInfo destinationBlobInfo = null;
string destinationStorageAccountConnectionString =
Environment.GetEnvironmentVariable("RES_STORAGE_ACCOUNT_CONN");
string sourceStorageAccountConnectionString =
Environment.GetEnvironmentVariable("SRC_STORAGE_ACCOUNT_CONN");
if (eventData is BlobEvent <CreatedEventData> )
{
// Retrieve the storage account from the connection string.
CloudStorageAccount sourceStorageAccount = CloudStorageAccount.Parse(sourceStorageAccountConnectionString);
CloudBlobClient sourceBlobClient = sourceStorageAccount.CreateCloudBlobClient();
// Retrieve the storage account from the connection string.
CloudStorageAccount destinationStorageAccount = CloudStorageAccount.Parse(destinationStorageAccountConnectionString);
CloudBlobClient destinationBlobClient = destinationStorageAccount.CreateCloudBlobClient();
BlobEvent <CreatedEventData> createdEventData = (BlobEvent <CreatedEventData>)eventData;
string url = createdEventData.data.url;
CloudBlockBlob sourceBlockBlob = new CloudBlockBlob(new Uri(url), sourceBlobClient);
bool sourceBlobExists = await sourceBlockBlob.ExistsAsync();
if (sourceBlobExists)
{
long blobSize = sourceBlockBlob.Properties.Length;
EventDateDetails dateDetails = new EventDateDetails(createdEventData.eventTime);
string destinationContaninerName = dateDetails.year.ToString();
string destinationBlobName = $"wk{dateDetails.WeekNumber}/dy{(int)dateDetails.DayOfWeek}/{sourceBlockBlob.Container.Name}/{sourceBlockBlob.Name}";
destinationBlobName += ".";
destinationBlobName += DateTimeUtil.GetString;
CloudBlobContainer destinationContainer = destinationBlobClient.GetContainerReference(destinationContaninerName);
bool result = await destinationContainer.CreateIfNotExistsAsync();
CloudBlockBlob destinationBlob = destinationContainer.GetBlockBlobReference(destinationBlobName);
_logger.LogInformation($"About to sync copy from Container: {sourceBlockBlob.Container.Name}, Blob: {sourceBlockBlob.Name}. Blob size: {sourceBlockBlob.Properties.Length} bytes");
// copyResult = "SYNCCOPY";
string copyResult = await destinationBlob.StartCopyAsync(sourceBlockBlob);
destinationBlobInfo = new DestinationBlobInfo();
destinationBlobInfo.ContainerName = destinationContainer.Name;
destinationBlobInfo.BlobName = destinationBlobName;
destinationBlobInfo.CopyReferenceId = copyResult;
destinationBlobInfo.OrgContainerName = sourceBlockBlob.Container.Name;
destinationBlobInfo.OrgBlobName = sourceBlockBlob.Name;
_logger.LogInformation($"Copy Scheduled. Source Blob Name: {sourceBlockBlob.Name}, Destination Blob Name: {destinationBlobInfo.BlobName}, Copy Id: {copyResult}.");
return(destinationBlobInfo);
}
else
{
_logger.LogInformation($"Not able to locate the block blob in source storage account---Block blob Name: {sourceBlockBlob.Name}");
}
}
else
{
_logger.LogInformation($"Input event data is not of Created Event Type.");
}
return(destinationBlobInfo);
}
/// <summary>
/// Run method
/// 1: Reads the messgaes from the queue.
/// 2: Stores the messages to the table storage.
/// 3: Deletes the messages from the queue
/// </summary>
/// <returns></returns>
public async Task Run()
{
_logger.LogDebug("Inside StorageBackup.Run");
var blobEvents = await _queueRepository.GetBLOBEvents();
_logger.LogInformation($"Number of messages found in queue.---{blobEvents.Count()}");
EventData eventData = null;
string eventString = string.Empty;
foreach (CloudQueueMessage blobEvent in blobEvents)
{
try
{
eventString = blobEvent.AsString;
eventData = new EventData(eventString);
if (eventData.RecievedEventData != null)
{
//In case file has been added, copy the file from source storage to destination storage
if (eventData.RecievedEventData is BlobEvent <CreatedEventData> )
{
_logger.LogDebug($"Going to write to blob---{@eventString}");
DestinationBlobInfo destinationBlobInfo = await _blobRepository.CopyBlobFromSourceToBackup(eventData.RecievedEventData);
eventData.DestinationBlobInfo = destinationBlobInfo;
eventData.DestinationBlobInfoJSON = JsonConvert.SerializeObject(destinationBlobInfo);
if (eventData.DestinationBlobInfo == null)
{
_logger.LogDebug($"DestinationBlobInfo is null. File not copied---{@eventString}");
}
}
else
{
_logger.LogDebug($"Skipping copying blob as it is not blob created event.---{@eventString}");
}
_logger.LogDebug($"Going to insert to storage---{@eventString}");
await _storageRepository.InsertBLOBEvent(eventData);
_logger.LogDebug($"Going to delete message from queue---{@eventString}");
//delete the message from queue after success insert only
await _queueRepository.DeleteBLOBEventAsync(blobEvent);
}
else
{
_logger.LogDebug($"EventData.RecievedEventData is null. Currently the utility understands Created and Deleted Events only.---{@eventString}");
}
}catch (Exception ex)
{
_logger.LogError($"Error while inserting to storage repository for this event. Event should come back to queue.Exception : {@ex.ToString()} Event Data :{@eventString}");
}
}
_logger.LogDebug("Completed StorageBackup.Run");
}
/// <summary>
/// CopyBlobFromSourceToBackup:
/// - Copies blob from source SA container to destination/backup SA container
/// </summary>
/// <returns></returns>
public async Task <DestinationBlobInfo> CopyBlobFromSourceToBackup(IBlobEvent eventData)
{
DestinationBlobInfo destinationBlobInfo = null;
string destinationStorageAccountConnectionString =
Environment.GetEnvironmentVariable("RES_STORAGE_ACCOUNT_CONN");
string sourceStorageAccountConnectionString =
Environment.GetEnvironmentVariable("SRC_STORAGE_ACCOUNT_CONN");
int blobTier =
string.IsNullOrEmpty(Environment.GetEnvironmentVariable("TargetBlobTier")) ?
2 :
int.Parse(Environment.GetEnvironmentVariable("TargetBlobTier"));
StandardBlobTier targetBlobTier; // Default tier is set to Cool
switch (blobTier)
{
case 1:
targetBlobTier = StandardBlobTier.Hot;
break;
case 2:
targetBlobTier = StandardBlobTier.Cool;
break;
case 3:
targetBlobTier = StandardBlobTier.Archive;
break;
default:
targetBlobTier = StandardBlobTier.Cool;
break;
}
;
if (eventData is BlobEvent <CreatedEventData> )
{
// Retrieve the storage account from the connection string.
CloudStorageAccount sourceStorageAccount = CloudStorageAccount.Parse(sourceStorageAccountConnectionString);
CloudBlobClient sourceBlobClient = sourceStorageAccount.CreateCloudBlobClient();
// Retrieve the storage account from the connection string.
CloudStorageAccount destinationStorageAccount = CloudStorageAccount.Parse(destinationStorageAccountConnectionString);
CloudBlobClient destinationBlobClient = destinationStorageAccount.CreateCloudBlobClient();
BlobEvent <CreatedEventData> createdEventData = (BlobEvent <CreatedEventData>)eventData;
string url = createdEventData.data.url;
CloudBlockBlob sourceBlockBlob = new CloudBlockBlob(new Uri(url), sourceBlobClient);
bool sourceBlobExists = await sourceBlockBlob.ExistsAsync();
if (sourceBlobExists)
{
long blobSize = sourceBlockBlob.Properties.Length;
EventDateDetails dateDetails = new EventDateDetails(createdEventData.eventTime);
string destinationContaninerName = dateDetails.year.ToString();
string destinationBlobName = $"wk{dateDetails.WeekNumber}/dy{(int)dateDetails.DayOfWeek}/{sourceBlockBlob.Container.Name}/{sourceBlockBlob.Name}";
destinationBlobName += ".";
destinationBlobName += DateTimeUtil.GetString;
CloudBlobContainer destinationContainer = destinationBlobClient.GetContainerReference(destinationContaninerName);
bool result = await destinationContainer.CreateIfNotExistsAsync();
CloudBlockBlob destinationBlob = destinationContainer.GetBlockBlobReference(destinationBlobName);
_logger.LogInformation($"About to sync copy from Container: {sourceBlockBlob.Container.Name}, Blob: {sourceBlockBlob.Name}. Blob size: {sourceBlockBlob.Properties.Length} bytes");
// copyResult = "SYNCCOPY";
string copyResult =
await destinationBlob.StartCopyAsync(
sourceBlockBlob,
targetBlobTier,
null, // Rehydrate priority
null, // Source access condition
null, // Destination access condition
null, // Blob request options
null, // Operation context
(new CancellationTokenSource()).Token);
destinationBlobInfo = new DestinationBlobInfo();
destinationBlobInfo.ContainerName = destinationContainer.Name;
destinationBlobInfo.BlobName = destinationBlobName;
destinationBlobInfo.CopyReferenceId = copyResult;
destinationBlobInfo.OrgContainerName = sourceBlockBlob.Container.Name;
destinationBlobInfo.OrgBlobName = sourceBlockBlob.Name;
_logger.LogInformation($"Copy Scheduled. Source Blob Name: {sourceBlockBlob.Name}, Destination Blob Name: {destinationBlobInfo.BlobName}, Copy Id: {copyResult}.");
return(destinationBlobInfo);
}
else
{
_logger.LogInformation($"Not able to locate the block blob in source storage account---Block blob Name: {sourceBlockBlob.Name}");
}
}
else
{
_logger.LogInformation($"Input event data is not of Created Event Type.");
}
return(destinationBlobInfo);
}
/// <summary>
/// CopyBlobFromBackupToRestore
/// - Copies storage blob from backup SA container to the respective source/restore SA container
/// </summary>
/// <returns></returns>
public async Task <string> CopyBlobFromBackupToRestore(DestinationBlobInfo backupBlob)
{
string destinationStorageAccountConnectionString =
Environment.GetEnvironmentVariable("SRC_STORAGE_ACCOUNT_CONN");
string sourceStorageAccountConnectionString =
Environment.GetEnvironmentVariable("RES_STORAGE_ACCOUNT_CONN");
int blobTier =
string.IsNullOrEmpty(Environment.GetEnvironmentVariable("TargetBlobTier")) ?
1 :
int.Parse(Environment.GetEnvironmentVariable("TargetBlobTier"));
StandardBlobTier targetBlobTier; // Default tier is set to Hot
switch (blobTier)
{
case 1:
targetBlobTier = StandardBlobTier.Hot;
break;
case 2:
targetBlobTier = StandardBlobTier.Cool;
break;
case 3:
targetBlobTier = StandardBlobTier.Archive;
break;
default:
targetBlobTier = StandardBlobTier.Hot;
break;
}
;
// Retrieve the storage account from the connection string.
CloudStorageAccount sourceStorageAccount = CloudStorageAccount.Parse(sourceStorageAccountConnectionString);
CloudBlobClient sourceBlobClient = sourceStorageAccount.CreateCloudBlobClient();
// Retrieve the storage account from the connection string.
CloudStorageAccount destinationStorageAccount = CloudStorageAccount.Parse(destinationStorageAccountConnectionString);
CloudBlobClient destinationBlobClient = destinationStorageAccount.CreateCloudBlobClient();
CloudBlobContainer sourceContainer = sourceBlobClient.GetContainerReference(backupBlob.ContainerName);
bool sourceContainerExists = await sourceContainer.ExistsAsync();
if (sourceContainerExists)
{
CloudBlockBlob sourceBlockBlob = sourceContainer.GetBlockBlobReference(backupBlob.BlobName);
bool sourceBlobExists = await sourceBlockBlob.ExistsAsync();
if (sourceBlobExists)
{
CloudBlobContainer destinationContainer = destinationBlobClient.GetContainerReference(backupBlob.OrgContainerName);
await destinationContainer.CreateIfNotExistsAsync();
CloudBlockBlob destinationBlob = destinationContainer.GetBlockBlobReference(backupBlob.OrgBlobName);
_logger.LogInformation($"About to sync copy from Container: {sourceBlockBlob.Container.Name}, Blob: {sourceBlockBlob.Name}. Blob size: {sourceBlockBlob.Properties.Length} bytes");
// copyResult = "SYNCCOPY";
string copyResult =
await destinationBlob.StartCopyAsync(
sourceBlockBlob,
targetBlobTier,
null, // Rehydrate priority
null, // Source access condition
null, // Destination access condition
null, // Blob request options
null, // Operation context
(new CancellationTokenSource()).Token);
_logger.LogInformation($"Copy Scheduled. Source Blob Name: {backupBlob.BlobName}, Destination Blob Name: {backupBlob.OrgBlobName}, Copy Id: {copyResult}.");
return(copyResult);
}
else
{
_logger.LogInformation($"Not able to locate the blob: {backupBlob.BlobName} in source storage account.");
}
}
else
{
_logger.LogInformation($"Not able to locate the container: {backupBlob.ContainerName} in source storage account.");
}
return(string.Empty);
}
/// <summary>
/// CopyBlobFromSourceToBackup
/// </summary>
/// <returns></returns>
public async Task <DestinationBlobInfo> CopyBlobFromSourceToBackup(IBlobEvent eventData)
{
DestinationBlobInfo destinationBlobInfo = null;
string destinationStorageAccountConnectionString = _config.GetConnectionString("BackupBlobStorage");
string sourceStorageAccountConnectionString = _config.GetConnectionString("SourceBlobStorage");
bool isServerCopy = bool.Parse(_config.GetSection("AppSettings")["IsServerCopy"]);
if (eventData is BlobEvent <CreatedEventData> )
{
// Retrieve the storage account from the connection string.
CloudStorageAccount sourceStorageAccount = CloudStorageAccount.Parse(sourceStorageAccountConnectionString);
CloudBlobClient sourceBlobClient = sourceStorageAccount.CreateCloudBlobClient();
// Retrieve the storage account from the connection string.
CloudStorageAccount destinationStorageAccount = CloudStorageAccount.Parse(destinationStorageAccountConnectionString);
CloudBlobClient destinationBlobClient = destinationStorageAccount.CreateCloudBlobClient();
BlobEvent <CreatedEventData> createdEventData = (BlobEvent <CreatedEventData>)eventData;
string url = createdEventData.data.url;
CloudBlockBlob sourceBlockBlob = new CloudBlockBlob(new Uri(url), sourceBlobClient);
bool sourceBlobExists = await sourceBlockBlob.ExistsAsync();
if (sourceBlobExists)
{
long blobSize = sourceBlockBlob.Properties.Length;
EventDateDetails dateDetails = new EventDateDetails(createdEventData.eventTime);
string destinationContaninerName = dateDetails.year.ToString();
string destinationBlobName = $"wk{dateDetails.WeekNumber}/dy{(int)dateDetails.DayOfWeek}/{sourceBlockBlob.Container.Name}/{sourceBlockBlob.Name}";
CloudBlobContainer destinationContainer = destinationBlobClient.GetContainerReference(destinationContaninerName);
bool result = await destinationContainer.CreateIfNotExistsAsync();
CloudBlockBlob destinationBlob = destinationContainer.GetBlockBlobReference(destinationBlobName);
string copyResult = string.Empty;
if (isServerCopy)
{
string blobToken = GenerateSASBlobToken(sourceBlockBlob);
_logger.LogInformation($"About to server copy {sourceBlockBlob.Name}. Blob size {sourceBlockBlob.Properties.Length} bytes");
copyResult = await destinationBlob.StartCopyAsync(new Uri(sourceBlockBlob.Uri.AbsoluteUri + blobToken));
}
else
{
_logger.LogInformation($"About to sync copy {sourceBlockBlob.Name}. Blob size {sourceBlockBlob.Properties.Length} bytes");
copyResult = "SYNCCOPY";
await TransferManager.CopyAsync(sourceBlockBlob, destinationBlob, false);
}
destinationBlobInfo = new DestinationBlobInfo();
destinationBlobInfo.ContainerName = destinationContainer.Name;
destinationBlobInfo.BlobName = destinationBlobName;
destinationBlobInfo.CopyReferenceId = copyResult;
destinationBlobInfo.OrgContainerName = sourceBlockBlob.Container.Name;
destinationBlobInfo.OrgBlobName = sourceBlockBlob.Name;
return(destinationBlobInfo);
}
else
{
_logger.LogInformation($"Not able to locate the block blob in source storage account---Block blob Name {sourceBlockBlob.Name}");
}
}
else
{
_logger.LogInformation($"Input event data is not of Created Event Type.");
}
return(destinationBlobInfo);
}
/// <summary>
/// CopyBlobFromBackupToRestore
/// </summary>
/// <returns></returns>
public async Task <string> CopyBlobFromBackupToRestore(DestinationBlobInfo backupBlob)
{
string destinationStorageAccountConnectionString = _config.GetConnectionString("RestoreBlobStorage");
string sourceStorageAccountConnectionString = _config.GetConnectionString("BackupBlobStorage");
bool isServerCopy = bool.Parse(_config.GetSection("AppSettings")["IsServerCopy"]);
// Retrieve the storage account from the connection string.
CloudStorageAccount sourceStorageAccount = CloudStorageAccount.Parse(sourceStorageAccountConnectionString);
CloudBlobClient sourceBlobClient = sourceStorageAccount.CreateCloudBlobClient();
// Retrieve the storage account from the connection string.
CloudStorageAccount destinationStorageAccount = CloudStorageAccount.Parse(destinationStorageAccountConnectionString);
CloudBlobClient destinationBlobClient = destinationStorageAccount.CreateCloudBlobClient();
CloudBlobContainer sourceContainer = sourceBlobClient.GetContainerReference(backupBlob.ContainerName);
bool sourceContainerExists = await sourceContainer.ExistsAsync();
if (sourceContainerExists)
{
CloudBlockBlob sourceBlockBlob = sourceContainer.GetBlockBlobReference(backupBlob.BlobName);
bool sourceBlobExists = await sourceBlockBlob.ExistsAsync();
if (sourceBlobExists)
{
CloudBlobContainer destinationContainer = destinationBlobClient.GetContainerReference(backupBlob.OrgContainerName);
await destinationContainer.CreateIfNotExistsAsync();
CloudBlockBlob destinationBlob = destinationContainer.GetBlockBlobReference(backupBlob.OrgBlobName);
string copyResult = string.Empty;
if (isServerCopy)
{
string blobToken = GenerateSASBlobToken(sourceBlockBlob);
_logger.LogInformation($"About to server copy {sourceBlockBlob.Name}. Blob size {sourceBlockBlob.Properties.Length} bytes");
copyResult = await destinationBlob.StartCopyAsync(new Uri(sourceBlockBlob.Uri.AbsoluteUri + blobToken));
}
else
{
_logger.LogInformation($"About to sync copy {sourceBlockBlob.Name}. Blob size {sourceBlockBlob.Properties.Length} bytes");
copyResult = "SYNCCOPY";
await TransferManager.CopyAsync(sourceBlockBlob, destinationBlob, false);
}
_logger.LogInformation($"Copy Scheduled. Source Blob Name: {backupBlob.BlobName} Destination Blob Name: {backupBlob.OrgBlobName} Copy Id {copyResult}.");
return(copyResult);
}
else
{
_logger.LogInformation($"Not able to locate the blob {backupBlob.BlobName} in source storage account.");
}
}
else
{
_logger.LogInformation($"Not able to locate the container {backupBlob.ContainerName} in source storage account.");
}
return(string.Empty);
}
