// OnDataLossAsync is the restore part of the process. This is NOT an arbitrary name, it is an override of the
// method on StatefulServiceBase.
// The method below looks in the backupFolder directory (where the backup files were
// stored in BackupCallbackAsync above), searches for the folder of backup files that have the newest ListWriteTime, then restores them.
// Once the RestoreDescription object is told where to find your data, RestoreAsync is called
// to restore that data to the reliable objects
// Each partition will have OnDataLossAsync called on it
protected override async Task <bool> OnDataLossAsync(RestoreContext restoreCtx,
CancellationToken cancellationToken)
{
string backupFolder;
ServiceEventSource.Current.ServiceMessage(this, "OnDataLoss Invoked!");
this.SetupBackupManager();
try
{
if (this.backupStorageType == BackupManagerType.None)
{
//since we have no backup configured, we return false to indicate
//that state has not changed. This replica will become the basis
//for future replica builds
return(false);
}
else
{
backupFolder =
await this.backupManager.RestoreLatestBackupToTempLocation(cancellationToken);
}
ServiceEventSource.Current.ServiceMessage(this, "Restoration Folder Path " + backupFolder);
RestoreDescription restoreDescription =
new RestoreDescription(backupFolder, RestorePolicy.Force);
// Restore the backup copy
await restoreCtx.RestoreAsync(restoreDescription, cancellationToken);
ServiceEventSource.Current.ServiceMessage(this, "Restore completed");
// Clean up the local temporary directory (the root directory where all backups were unzipped in to)
try
{
DirectoryInfo tempRestoreDirectory = new DirectoryInfo(backupFolder);
tempRestoreDirectory.Delete(true);
}
catch (System.Exception ddel)
{
ServiceEventSource.Current.ServiceMessage(this, "OnDataLossAsync: Delete of backup folder failed with {0}", ddel.Message);
}
// now that a restore has been performed, reset the full backup flag and incremental
// counter so you start fresh again
// You will need to change this type of logic for your own scenario
takeFullBackup = true;
incrementalCount = 0;
return(true);
}
catch (Exception e)
{
ServiceEventSource.Current.ServiceMessage(this, "Restoration failed: " + "{0} {1}" + e.GetType() + e.Message);
throw;
}
}
protected override async Task <bool> OnDataLossAsync(RestoreContext restoreCtx, CancellationToken cancellationToken)
{
ServiceEventSource.Current.ServiceMessage(this.Context, "OnDataLoss Invoked!");
this.SetupBackupManager();
try
{
string backupFolder;
backupFolder = await this.backupManager.RestoreLatestBackupToTempLocation(cancellationToken);
ServiceEventSource.Current.ServiceMessage(this.Context, "Restoration Folder Path " + backupFolder);
RestoreDescription restoreRescription = new RestoreDescription(backupFolder, RestorePolicy.Force);
await restoreCtx.RestoreAsync(restoreRescription, cancellationToken);
ServiceEventSource.Current.ServiceMessage(this.Context, "Restore completed");
DirectoryInfo tempRestoreDirectory = new DirectoryInfo(backupFolder);
tempRestoreDirectory.Delete(true);
return(true);
}
catch (Exception e)
{
ServiceEventSource.Current.ServiceMessage(this.Context, "Restoration failed: " + "{0} {1}" + e.GetType() + e.Message);
throw;
}
}
protected override async Task <bool> OnDataLossAsync(RestoreContext restoreCtx, CancellationToken cancellationToken)
{
var backupsourcedir = this._fileStore.GetRestoreDirectory();
this._serviceEventSource.Message($"BackupRestoreStatefulService: Restoring backup of partition { Context.PartitionId} of service {Context.ServiceName.AbsoluteUri} from {backupsourcedir}");
await restoreCtx.RestoreAsync(new RestoreDescription(backupsourcedir, RestorePolicy.Force), cancellationToken);
this._serviceEventSource.Message($"BackupRestoreStatefulService: Backup of partition { Context.PartitionId} of service {Context.ServiceName.AbsoluteUri} from {backupsourcedir} has been restored!");
return(true);
}
//Dataloss testing can be triggered via powershell. To do so, run the following commands as a script
//Connect-ServiceFabricCluster
//$s = "fabric:/WebReferenceApplication/InventoryService"
//$p = Get-ServiceFabricApplication | Get-ServiceFabricService -ServiceName $s | Get-ServiceFabricPartition | Select -First 1
//$p | Invoke-ServiceFabricPartitionDataLoss -DataLossMode FullDataLoss -ServiceName $s
protected override async Task <bool> OnDataLossAsync(RestoreContext restoreCtx, CancellationToken cancellationToken)
{
ServiceEventSource.Current.ServiceMessage(this, "OnDataLoss Invoked!");
this.SetupBackupManager();
try
{
string backupFolder;
if (this.backupStorageType == BackupManagerType.None)
{
//since we have no backup configured, we return false to indicate
//that state has not changed. This replica will become the basis
//for future replica builds
return(false);
}
else
{
backupFolder = await this.backupManager.RestoreLatestBackupToTempLocation(cancellationToken);
}
ServiceEventSource.Current.ServiceMessage(this, "Restoration Folder Path " + backupFolder);
RestoreDescription restoreRescription = new RestoreDescription(backupFolder, RestorePolicy.Force);
await restoreCtx.RestoreAsync(restoreRescription, cancellationToken);
ServiceEventSource.Current.ServiceMessage(this, "Restore completed");
DirectoryInfo tempRestoreDirectory = new DirectoryInfo(backupFolder);
tempRestoreDirectory.Delete(true);
return(true);
}
catch (Exception e)
{
ServiceEventSource.Current.ServiceMessage(this, "Restoration failed: " + "{0} {1}" + e.GetType() + e.Message);
throw;
}
}
/// <summary>
/// This method is called during suspected data loss.
/// You can use this method to restore the service in case of data loss.
/// </summary>
/// <param name="service"></param>
/// <param name="restoreCtx"></param>
/// <param name="cancellationToken"></param>
/// <returns></returns>
public static async Task <bool> OnDataLossAsync(this IBackupRestoreServiceInternal service, RestoreContext restoreCtx, CancellationToken cancellationToken)
{
//caused by BeginRestoreBackup
service.LogCallback?.Invoke($"BackupRestoreService - OnDataLossAsync starting for partition: {service.Context.PartitionId}.");
var metadata = await service.CentralBackupStore.RetrieveScheduledBackupAsync(service.Context.PartitionId);
if (metadata == null)
{
return(false);
}
List <BackupMetadata> backupList = await GetBackupMetadataAsync(service, metadata);
if (backupList.Count == 0)
{
throw new InvalidOperationException("Failed to find any backups for this partition.");
}
if (backupList[0].BackupOption != BackupOption.Full)
{
throw new InvalidOperationException("Failed to find any full backups for this partition.");
}
string localBackupFolder = Path.Combine(service.Context.CodePackageActivationContext.WorkDirectory, Guid.NewGuid().ToString("N"));
foreach (var backupMetadata in backupList)
{
string subFolder = Path.Combine(localBackupFolder, backupMetadata.TimeStampUtc.ToString("yyyyMMddhhmmss"));
await service.CentralBackupStore.DownloadBackupFolderAsync(backupMetadata.BackupId, subFolder, cancellationToken);
}
var restoreDescription = new RestoreDescription(localBackupFolder, RestorePolicy.Force);
await restoreCtx.RestoreAsync(restoreDescription, cancellationToken);
Directory.Delete(localBackupFolder, true);
service.LogCallback?.Invoke($"BackupRestoreService - OnDataLossAsync complete for partition {service.Context.PartitionId}.");
return(true);
}
protected override async Task <bool> OnDataLossAsync(RestoreContext restoreCtx, CancellationToken cancellationToken)
{
this.SetupBackupManager();
try
{
string backupFolder;
backupFolder = await this.backupManager.RestoreLatestBackupToTempLocation(cancellationToken);
RestoreDescription restoreRescription = new RestoreDescription(backupFolder, RestorePolicy.Force);
await restoreCtx.RestoreAsync(restoreRescription, cancellationToken);
DirectoryInfo tempRestoreDirectory = new DirectoryInfo(backupFolder);
tempRestoreDirectory.Delete(true);
return(true);
}
catch (Exception e)
{
throw;
}
}
// OnDataLossAsync is the restore part of the process. This is NOT an arbitrary name, it is an override of the
// method on StatefulServiceBase.
// The method below looks in the c:\temp\sfbackup directory (where I stored the backup files in BackupCallbackAsync
// above), searches for the folder of backup files that have the newest ListWriteTime, then restore them.
// Once the RestoreDescription object is told where to find your data, RestoreAsync is called
// to restore that data to the reliable objects
protected override async Task <bool> OnDataLossAsync(RestoreContext restoreCtx,
CancellationToken cancellationToken)
{
ServiceEventSource.Current.ServiceMessage(this, "OnDataLoss Invoked!");
this.SetupBackupManager();
try
{
string restorePath = "";
if (this.backupStorageType == BackupManagerType.None)
{
//since we have no backup configured, we return false to indicate
//that state has not changed. This replica will become the basis
//for future replica builds
return(false);
}
else
{
string zipPath = "";
string partitionId = this.Context.PartitionId.ToString("N");
//this will be the root directory for all unzipped files
ICodePackageActivationContext codePackageContext = this.Context.CodePackageActivationContext;
//this.PartitionTempDirectory = Path.Combine(codePackageContext.TempDirectory, partitionId);
this.PartitionTempDirectory = Path.Combine(@"C:\myrestore", partitionId);
// Get back the full list of blobs and do not sort them
// Sorting will be used when restoring a full-only backup
IEnumerable <IListBlobItem> blobs = await this.backupManager.GetBackupBlobs(false);
// the partition temp directory is the root restore directory + partitionId
// create this directory
System.IO.Directory.CreateDirectory(this.PartitionTempDirectory);
foreach (CloudBlockBlob backup in blobs)
{
// this is the full zip path including file name and extension
// this is the location the zip file should be downloaded to
zipPath = Path.Combine(this.PartitionTempDirectory, backup.Name);
// download the zip file from storage
try
{
// download the file in to the partition directory
backup.DownloadToFile(zipPath, FileMode.CreateNew);
}
catch (System.Exception ee)
{
}
string restoreDir = (backup.Name).Remove(backup.Name.Length - 4);
// Create a path in which to unzip the zip file that has been downloaded
// to the partiton directory
//restorePath = Path.Combine(this.PartitionTempDirectory, partitionId);
restorePath = Path.Combine(this.PartitionTempDirectory, restoreDir);
//extract the zip in to the local temp path directory
ZipFile.ExtractToDirectory(zipPath, restorePath);
}
//clean up the zip file, files are now in the restore path
FileInfo zipInfo = new FileInfo(zipPath);
zipInfo.Delete();
//backupFolder =
// await this.backupManager.RestoreLatestBackupToTempLocationIncremental(cancellationToken);
}
ServiceEventSource.Current.ServiceMessage(this, "Restoration Folder Path " + restorePath);
RestoreDescription restoreDescription =
new RestoreDescription(this.PartitionTempDirectory, RestorePolicy.Force);
// Restore the backup copy
await restoreCtx.RestoreAsync(restoreDescription, cancellationToken);
ServiceEventSource.Current.ServiceMessage(this, "Restore completed");
// Clean up the local temporary directory
DirectoryInfo tempRestoreDirectory = new DirectoryInfo(this.PartitionTempDirectory);
tempRestoreDirectory.Delete(true);
return(true);
}
catch (Exception e)
{
ServiceEventSource.Current.ServiceMessage(this, "Restoration failed: " + "{0} {1}" + e.GetType() + e.Message);
throw;
}
}
//Dataloss testing can be triggered via powershell. To do so, run the following commands as a script
//Connect-ServiceFabricCluster
//$s = "fabric:/WebReferenceApplication/InventoryService"
//$p = Get-ServiceFabricApplication | Get-ServiceFabricService -ServiceName $s | Get-ServiceFabricPartition | Select -First 1
//$p | Invoke-ServiceFabricPartitionDataLoss -DataLossMode FullDataLoss -ServiceName $s
protected override async Task<bool> OnDataLossAsync(RestoreContext restoreCtx, CancellationToken cancellationToken)
{
ServiceEventSource.Current.ServiceMessage(this, "OnDataLoss Invoked!");
this.SetupBackupManager();
try
{
string backupFolder;
if (this.backupStorageType == BackupManagerType.None)
{
//since we have no backup configured, we return false to indicate
//that state has not changed. This replica will become the basis
//for future replica builds
return false;
}
else
{
backupFolder = await this.backupManager.RestoreLatestBackupToTempLocation(cancellationToken);
}
ServiceEventSource.Current.ServiceMessage(this, "Restoration Folder Path " + backupFolder);
RestoreDescription restoreRescription = new RestoreDescription(backupFolder, RestorePolicy.Force);
await restoreCtx.RestoreAsync(restoreRescription, cancellationToken);
ServiceEventSource.Current.ServiceMessage(this, "Restore completed");
DirectoryInfo tempRestoreDirectory = new DirectoryInfo(backupFolder);
tempRestoreDirectory.Delete(true);
return true;
}
catch (Exception e)
{
ServiceEventSource.Current.ServiceMessage(this, "Restoration failed: " + "{0} {1}" + e.GetType() + e.Message);
throw;
}
}
/// <summary>
/// This method is called during suspected data loss.
/// You can use this method to restore the service in case of data loss.
/// </summary>
/// <param name="service"></param>
/// <param name="restoreCtx"></param>
/// <param name="cancellationToken"></param>
/// <returns></returns>
public static async Task <bool> OnDataLossAsync(this IBackupRestoreServiceOperations service, RestoreContext restoreCtx, CancellationToken cancellationToken)
{
//Ususally caused by BeginRestoreBackup
//First, query the scheduled metadata for this partition backup
service.LogCallback?.Invoke($"{nameof(BackupRestoreServiceOperations)} - {nameof(OnDataLossAsync)} - Starting for partition: {service.Context.PartitionId}.");
BackupMetadata metadata;
try
{
metadata = await service.CentralBackupStore.RetrieveScheduledBackupAsync(service.Context.PartitionId);
}
catch (Exception ex)
{
string message = $"Failed to find backup information for partition {service.Context.PartitionId}";
service.LogCallback?.Invoke($"{nameof(BackupRestoreServiceOperations)} - {nameof(OnDataLossAsync)} - Failed for partition: {service.Context.PartitionId}. Message:{message} - Error: {ex.Message}");
throw new Exception(message, ex);
}
if (metadata == null)
{
return(false);
}
//Get all metadata for all backups related to this one, if it's an incremental one
List <BackupMetadata> backupList;
try
{
backupList = await GetBackupMetadataAsync(service, metadata);
if (backupList.Count < 1)
{
throw new InvalidOperationException("Failed to find any backups for this partition.");
}
if (backupList[0].BackupOption != BackupOption.Full)
{
throw new InvalidOperationException("Failed to find any full backups for this partition.");
}
}
catch (Exception ex)
{
string message = $"Failed to find backup information for partition {service.Context.PartitionId}";
service.LogCallback?.Invoke($"{nameof(BackupRestoreServiceOperations)} - {nameof(OnDataLossAsync)} - Failed for partition: {service.Context.PartitionId}. Message:{message} - Error: {ex.Message}");
throw new Exception(message, ex);
}
//download central to local
string localBackupFolder = Path.Combine(service.Context.CodePackageActivationContext.WorkDirectory, Guid.NewGuid().ToString("N"));
try
{
foreach (var backupMetadata in backupList)
{
string subFolder = Path.Combine(localBackupFolder, backupMetadata.TimeStampUtc.ToString("yyyyMMddhhmmss"));
await service.CentralBackupStore.DownloadBackupFolderAsync(backupMetadata.BackupId, subFolder, cancellationToken);
}
}
catch (Exception ex)
{
string message = $"Failed to download backup data for partition {service.Context.PartitionId}";
service.LogCallback?.Invoke($"{nameof(BackupRestoreServiceOperations)} - {nameof(OnDataLossAsync)} - Failed for partition: {service.Context.PartitionId}. Message:{message} - Error: {ex.Message}");
throw new Exception(message, ex);
}
//restore
try
{
var restoreDescription = new RestoreDescription(localBackupFolder, RestorePolicy.Force);
await restoreCtx.RestoreAsync(restoreDescription, cancellationToken);
}
catch (Exception ex)
{
string message = $"Failed to restore backup for partition {service.Context.PartitionId}";
service.LogCallback?.Invoke($"{nameof(BackupRestoreServiceOperations)} - {nameof(OnDataLossAsync)} - Failed for partition: {service.Context.PartitionId}. Message:{message} - Error: {ex.Message}");
throw new Exception(message, ex);
}
finally
{
Directory.Delete(localBackupFolder, true);
}
//success!!
service.LogCallback?.Invoke($"BackupRestoreService - OnDataLossAsync complete for partition {service.Context.PartitionId}.");
return(true);
}