/// <inheritdoc />
public async Task <BackupMetadata> RetrieveScheduledBackupAsync(Guid servicePartitionId)
{
if (!_isInitialized)
{
await InitializeAsync().ConfigureAwait(false);
}
BackupMetadata backup = null;
//retrieve the backup to restore for the provided partition
var directoryReference = _blobContainer.GetDirectoryReference($"{RootFolder}/{QueueFolder}");
var fileReference = directoryReference.GetBlockBlobReference(servicePartitionId.ToString("N"));
if (!(await fileReference.ExistsAsync()))
{
throw new Exception($"Backup for partition {servicePartitionId:N} was not found in the queue.");
}
string content = await fileReference.DownloadTextAsync();
Guid id;
if (Guid.TryParse(content, out id))
{
backup = (await GetBackupMetadataAsync(id)).SingleOrDefault();
if (!(await fileReference.ExistsAsync()))
{
throw new Exception($"Backup for partition {servicePartitionId:N} with id {id:N} was not found in the metadata. (Corruption)");
}
}
await fileReference.DeleteAsync();
return(backup);
}
/// <inheritdoc />
public Task StoreBackupMetadataAsync(string destinationFolder, BackupMetadata info)
{
return(Task.Run(() =>
{
string file = Path.Combine(destinationFolder, ServiceFabricBackupRestoreMetadataFileName);
string json = JsonConvert.SerializeObject(info);
File.WriteAllText(file, json);
}));
}
/// <inheritdoc />
public async Task <BackupMetadata> UploadBackupFolderAsync(BackupOption backupOption, Guid servicePartitionId, string sourceDirectory,
CancellationToken cancellationToken)
{
//use folder names containing the service partition and the utc date:
var timeStamp = DateTime.UtcNow;
string destinationFolder = CreateDateTimeFolderName(servicePartitionId, timeStamp);
//upload
await CopyFolderAsync(sourceDirectory, destinationFolder, cancellationToken);
//create metadata to return
var info = new BackupMetadata(servicePartitionId, timeStamp, backupOption);
//store the backup id.
await StoreBackupMetadataAsync(destinationFolder, info);
return(info);
}
/// <summary>
/// Stores the provided metadata as blob and as blob metadata.
/// </summary>
/// <param name="destinationFolder"></param>
/// <param name="info"></param>
/// <param name="cancellationToken"></param>
/// <returns></returns>
public async Task StoreBackupMetadataAsync(string destinationFolder, BackupMetadata info, CancellationToken cancellationToken = default(CancellationToken))
{
if (!_isInitialized)
{
await InitializeAsync().ConfigureAwait(false);
}
string destFileName = $"{destinationFolder}/{FileStore.ServiceFabricBackupRestoreMetadataFileName}";
string json = JsonConvert.SerializeObject(info);
var blockBlobReference = _blobContainer.GetBlockBlobReference(destFileName);
await blockBlobReference.UploadTextAsync(json, null, null, null, null, cancellationToken);
blockBlobReference.Metadata.Add(nameof(BackupMetadata.BackupId), info.BackupId.ToString("N"));
blockBlobReference.Metadata.Add(nameof(BackupMetadata.BackupOption), info.BackupOption.ToString());
blockBlobReference.Metadata.Add(nameof(BackupMetadata.OriginalServicePartitionId), info.OriginalServicePartitionId.ToString("N"));
blockBlobReference.Metadata.Add(nameof(BackupMetadata.TimeStampUtc), info.TimeStampUtc.ToString("o"));
await blockBlobReference.SetMetadataAsync();
}
/// <summary>
/// Compiles a list of <see cref="BackupMetadata"/> to be restored, based on the provided metadata.
/// </summary>
/// <param name="service"></param>
/// <param name="metadata"></param>
/// <returns></returns>
internal static async Task <List <BackupMetadata> > GetBackupMetadataAsync(IBackupRestoreServiceInternal service, BackupMetadata metadata)
{
var backupList = new List <BackupMetadata>();
if (metadata.BackupOption == BackupOption.Full)
{
// Taking only selected full backup
backupList.Add(metadata);
}
else
{
// Taking full backup and all incremental backups since, reversed
// so it finds the incremental backup first, and then loops until the full backup is found.
var allBackups = (await service.CentralBackupStore.GetBackupMetadataAsync(servicePartitionId: metadata.OriginalServicePartitionId))
.OrderByDescending(x => x.TimeStampUtc)
.ToList();
BackupMetadata incrementalBackup = null;
// Looking for the latest full backup before selected
foreach (var backupMetadata in allBackups)
{
if (incrementalBackup == null && backupMetadata.BackupId != metadata.BackupId)
{
continue;
}
else
{
incrementalBackup = backupMetadata;
backupList.Add(backupMetadata);
}
if (backupMetadata.BackupOption == BackupOption.Full)
{
//if it's the full backup we encounter, we're done
break;
}
}
backupList.Reverse();
}
return(backupList);
}
/// <inheritdoc />
public async Task <BackupMetadata> RetrieveScheduledBackupAsync(Guid servicePartitionId)
{
BackupMetadata backup = null;
//retrieve the backup to restore for the provided partition
string queueFile = GetQueueFile(servicePartitionId);
if (!File.Exists(queueFile))
{
return(null);
}
string content = File.ReadAllText(queueFile);
Guid id;
if (Guid.TryParse(content, out id))
{
backup = (await GetBackupMetadataAsync(id)).Single();
}
File.Delete(queueFile);
return(backup);
}
/// <inheritdoc />
public Task BeginRestoreBackup(BackupMetadata backupMetadata, DataLossMode dataLossMode)
{
return(BackupRestoreServiceOperations.BeginRestoreBackup(this, backupMetadata, dataLossMode));
}
/// <inheritdoc />
public Task BeginRestoreBackup(BackupMetadata backupMetadata, DataLossMode dataLossMode)
{
return(BackupRestoreServiceInternalExtensions.BeginRestoreBackup(this, backupMetadata, dataLossMode));
}
/// <summary>
/// Asynchronously starts a restore operation using the state indicated by <paramref name="backupMetadata"/>.
/// The backup is retrieved from the central store.
/// </summary>
/// <param name="service"></param>
/// <param name="dataLossMode"></param>
/// <param name="backupMetadata"></param>
/// <returns></returns>
public static async Task BeginRestoreBackup(this IBackupRestoreServiceInternal service, BackupMetadata backupMetadata, DataLossMode dataLossMode)
{
service.LogCallback?.Invoke($"BackupRestoreService - Beginning restore backup {backupMetadata.BackupId} for partition {service.Context.PartitionId}.");
if (backupMetadata == null)
{
throw new ArgumentNullException(nameof(backupMetadata));
}
await service.CentralBackupStore.ScheduleBackupAsync(service.Context.PartitionId, backupMetadata.BackupId);
var partitionSelector = PartitionSelector.PartitionIdOf(service.Context.ServiceName, service.Context.PartitionId);
var operationId = Guid.NewGuid();
await new FabricClient(FabricClientRole.Admin).TestManager.StartPartitionDataLossAsync(operationId, partitionSelector, dataLossMode);
//Causes OnDataLossAsync to be called.
service.LogCallback?.Invoke($"BackupRestoreService - Begun restore backup {backupMetadata.BackupId} for partition {service.Context.PartitionId}.");
}
/// <summary>
/// Asynchronously starts a restore operation using the state indicated by <paramref name="backupMetadata"/>.
/// The backup is retrieved from the central store.
/// This method completes and returns before the backup restore process is completely done.
/// </summary>
/// <param name="service"></param>
/// <param name="dataLossMode"></param>
/// <param name="backupMetadata"></param>
/// <returns></returns>
public static async Task BeginRestoreBackup(this IBackupRestoreServiceOperations service, BackupMetadata backupMetadata, DataLossMode dataLossMode)
{
service.LogCallback?.Invoke($"BackupRestoreService - Beginning restore backup {backupMetadata.BackupId} for partition {service.Context.PartitionId}.");
try
{
if (backupMetadata == null)
{
throw new ArgumentNullException(nameof(backupMetadata));
}
await service.CentralBackupStore.ScheduleBackupRestoreAsync(service.Context.PartitionId, backupMetadata.BackupId);
var partitionSelector = PartitionSelector.PartitionIdOf(service.Context.ServiceName, service.Context.PartitionId);
var operationId = Guid.NewGuid();
await new FabricClient(FabricClientRole.Admin).TestManager.StartPartitionDataLossAsync(operationId, partitionSelector, dataLossMode);
//Causes OnDataLossAsync to be called later on.
}
catch (Exception ex)
{
string message = $"Failed to restore backup for partition {service.Context.PartitionId}";
service.LogCallback?.Invoke($"{nameof(BackupRestoreServiceOperations)} - {nameof(BeginRestoreBackup)} failed for partition: {service.Context.PartitionId}. Message:{message} - Error: {ex.Message}");
throw new Exception(message, ex);
}
service.LogCallback?.Invoke($"{nameof(BackupRestoreServiceOperations)} - {nameof(BeginRestoreBackup)} succeeded {backupMetadata.BackupId} for partition {service.Context.PartitionId}.");
}
