public async Task <List <DynamicTableEntity> > GetEntitiesAsync(IAzureTableBackup backupSettings)
{
var table = await GetCloudTableAsync(backupSettings.SourceConnectionString, backupSettings.SourceTableName);
var entities = new List <DynamicTableEntity>();
var query = table.CreateQuery <DynamicTableEntity>().AsTableQuery();
if (!(await table.ExistsAsync()))
{
await _loggingRepository.LogMessageAsync(new LogMessage { Message = $"Source table {backupSettings.SourceTableName} was not found!" });
return(null);
}
TableContinuationToken continuationToken = null;
do
{
var segmentResult = await table.ExecuteQuerySegmentedAsync(query, continuationToken);
continuationToken = segmentResult.ContinuationToken;
entities.AddRange(segmentResult.Results);
} while (continuationToken != null);
return(entities);
}
private async Task BackupTableAsync(IAzureTableBackup table)
{
await _loggingRepository.LogMessageAsync(new LogMessage { Message = $"Starting backup maintenance for table: {table.SourceTableName}" });
var entities = await _azureTableBackupRepository.GetEntitiesAsync(table);
if (entities == null)
{
return;
}
// basically, the table storage gets used regardless, to read the source table
// and to maintain the tracking table. this determines whether the backup data
// is stored in table or blob storage.
IAzureStorageBackupRepository backUpTo = DetermineBackupStorage(table.BackupType);
if (backUpTo == null)
{
await _loggingRepository.LogMessageAsync(new LogMessage { Message = $"BackupType must be 'table' or 'blob'. Was {table.BackupType}. Not backing up {table.SourceTableName}." });
return;
}
await _loggingRepository.LogMessageAsync(new LogMessage { Message = $"Backing up {entities.Count} entites from table {table.SourceTableName} to {table.BackupType} storage." });
var backupResult = await backUpTo.BackupEntitiesAsync(table, entities);
await CompareBackupResults(table, backupResult);
}
public async Task <List <BackupEntity> > GetBackupsAsync(IAzureTableBackup backupSettings)
{
var blobContainerClient = await GetContainerClient(backupSettings);
if (blobContainerClient == null)
{
return(new List <BackupEntity>());
}
var backupEntities = new List <BackupEntity>();
backupEntities.Add(new BackupEntity(GetContainerName(backupSettings), "blob"));
return(backupEntities);
}
private async Task <BlobContainerClient> GetContainerClient(IAzureTableBackup backupSettings)
{
BlobServiceClient blobServiceClient = new BlobServiceClient(backupSettings.DestinationConnectionString);
var containerName = GetContainerName(backupSettings);
var blobClient = blobServiceClient.GetBlobContainerClient(containerName);
if (!await blobClient.ExistsAsync())
{
await _loggingRepository.LogMessageAsync(new LogMessage { Message = $"The blob container {containerName} does not exist." });
return(null);
}
return(blobClient);
}
public async Task DeleteBackupAsync(IAzureTableBackup backupSettings, string backupName)
{
var blobContainerClient = await GetContainerClient(backupSettings);
if (blobContainerClient == null)
{
await _loggingRepository.LogMessageAsync(new LogMessage { Message = $"The blob container {GetContainerName(backupSettings)} did not exist when trying to delete blob {backupName}." });
return;
}
// This only gives us a "true" or a "false". I assume a "false" means the blob didn't exist.
// I think the only way this could try to delete a blob that doesn't exist is if there's some kind of weird race condition
// that leads to a blob getting deleted twice, and that's fine, too.
var response = await blobContainerClient.DeleteBlobIfExistsAsync(backupName);
await _loggingRepository.LogMessageAsync(new LogMessage { Message = $"Got {response.Value} when deleting blob {backupName}." });
}
public async Task <bool> ReviewAndDeleteAsync(IAzureTableBackup backupSetting, string tableName)
{
var backupStorage = DetermineBackupStorage(backupSetting.BackupType);
var shouldDelete = await backupStorage.VerifyDeleteBackupAsync(backupSetting, tableName);
if (shouldDelete)
{
await backupStorage.DeleteBackupAsync(backupSetting, tableName);
if (!backupSetting.CleanupOnly && backupSetting.BackupType.ToLowerInvariant() == "table")
{
await DeleteOldBackupTrackersAsync(backupSetting, new List <string> {
tableName
});
}
}
return(shouldDelete);
}
public async Task <List <BackupEntity> > GetBackupsAsync(IAzureTableBackup backupSettings)
{
await _loggingRepository.LogMessageAsync(new LogMessage { Message = $"Getting backup tables for table {backupSettings.SourceTableName}" });
var storageAccount = CloudStorageAccount.Parse(backupSettings.DestinationConnectionString);
var tableClient = storageAccount.CreateCloudTableClient();
List <CloudTable> tables;
if (backupSettings.SourceTableName == "*")
{
tables = tableClient.ListTables().ToList();
}
else
{
tables = tableClient.ListTables(prefix: BACKUP_PREFIX + backupSettings.SourceTableName).ToList();
}
await _loggingRepository.LogMessageAsync(new LogMessage { Message = $"Found {tables.Count} backup tables for table {backupSettings.SourceTableName}" });
return(tables.Select(table => new BackupEntity(table.Name, "table")).ToList());
}
public async Task <BackupResult> BackupEntitiesAsync(IAzureTableBackup backupSettings, List <DynamicTableEntity> entities)
{
if (!entities.Any())
{
await _loggingRepository.LogMessageAsync(new LogMessage { Message = $"Nothing to back up for {backupSettings.SourceTableName}. Skipping." });
}
BlobServiceClient blobServiceClient = new BlobServiceClient(backupSettings.DestinationConnectionString);
string containerName = GetContainerName(backupSettings);
var blobName = $"{BACKUP_PREFIX}{backupSettings.SourceTableName}{DateTime.UtcNow.ToString(BACKUP_DATE_FORMAT)}.csv";
await _loggingRepository.LogMessageAsync(
new LogMessage
{
Message = $"Backing up data to blob: {blobName} started",
DynamicProperties =
{
{ "status", "Started" },
{ "rowCount", entities.Count.ToString() }
}
});
var blobClient = blobServiceClient.GetBlobContainerClient(containerName);
await blobClient.CreateIfNotExistsAsync();
var result = await blobClient.UploadBlobAsync(blobName, new BinaryData(SerializeEntities(entities)));
await _loggingRepository.LogMessageAsync(
new LogMessage
{
Message = $"Backing up data to blob: {blobName} completed",
DynamicProperties =
{
{ "status", "Completed" },
{ "rowCount", entities.Count.ToString() }
}
});
return(new BackupResult(blobName, "blob", entities.Count));
}
private async Task CompareBackupResults(IAzureTableBackup backupSettings, BackupResult currentBackup)
{
var previousBackupTracker = await _azureTableBackupRepository.GetLastestBackupResultTrackerAsync(backupSettings);
await _azureTableBackupRepository.AddBackupResultTrackerAsync(backupSettings, currentBackup);
if (previousBackupTracker == null)
{
return;
}
var delta = currentBackup.RowCount - previousBackupTracker.RowCount;
var message = delta == 0 ? " same number of" : ((delta > 0) ? " more" : " less");
await _loggingRepository.LogMessageAsync(
new LogMessage
{
Message = $"Current backup for {backupSettings.SourceTableName} has {delta}{message} rows than previous backup",
DynamicProperties =
{
{ "status", "Delta" },
{ "rowCount", delta.ToString() }
}
});
}
public async Task <bool> VerifyDeleteBackupAsync(IAzureTableBackup backupSettings, string blobName)
{
var cutOffDate = DateTime.UtcNow.AddDays(-backupSettings.DeleteAfterDays);
var blobContainerClient = await GetContainerClient(backupSettings);
var blobClient = blobContainerClient.GetBlobClient(blobName);
try
{
var parsedDateTimeOffset = DateTimeOffset.ParseExact(blobName.Replace(".csv", string.Empty).Replace(BACKUP_PREFIX + backupSettings.SourceTableName, string.Empty),
BACKUP_DATE_FORMAT,
null,
DateTimeStyles.AssumeUniversal);
var blobProperties = await blobClient.GetPropertiesAsync();
var CreatedDate = parsedDateTimeOffset.UtcDateTime;
// Use the blob property if it can be retrieved, otherwise parse time
if (blobProperties.Value != null)
{
CreatedDate = blobProperties.Value.CreatedOn.UtcDateTime;
}
if (CreatedDate != null && CreatedDate < cutOffDate)
{
return(true);
}
}
catch
{
await _loggingRepository.LogMessageAsync(new LogMessage { Message = $"The blob item {blobName} did not have a CreatedOn property and its name was unparseable" });
}
return(false);
}
private async Task DeleteOldBackupTrackersAsync(IAzureTableBackup backupSettings, List <string> deletedTables)
{
var keys = deletedTables.Select(x => (backupSettings.SourceTableName, x)).ToList();
await _azureTableBackupRepository.DeleteBackupTrackersAsync(backupSettings, keys);
}
public async Task DeleteBackupTrackersAsync(IAzureTableBackup backupSettings, List <(string PartitionKey, string RowKey)> entities)
private static string GetContainerName(IAzureTableBackup backupSettings)
{
return($"{BACKUP_PREFIX}{backupSettings.SourceTableName}".ToLowerInvariant());
}
