protected override async Task <bool> OnDataLossAsync(RestoreContext restoreCtx, CancellationToken cancellationToken)
{
ServiceEventSource.Current.ServiceMessage(this.Context, "OnDataLoss Invoked!");
this.SetupBackupManager();
try
{
string backupFolder;
backupFolder = await this.backupManager.RestoreLatestBackupToTempLocation(cancellationToken);
ServiceEventSource.Current.ServiceMessage(this.Context, "Restoration Folder Path " + backupFolder);
RestoreDescription restoreRescription = new RestoreDescription(backupFolder, RestorePolicy.Force);
await restoreCtx.RestoreAsync(restoreRescription, cancellationToken);
ServiceEventSource.Current.ServiceMessage(this.Context, "Restore completed");
DirectoryInfo tempRestoreDirectory = new DirectoryInfo(backupFolder);
tempRestoreDirectory.Delete(true);
return(true);
}
catch (Exception e)
{
ServiceEventSource.Current.ServiceMessage(this.Context, "Restoration failed: " + "{0} {1}" + e.GetType() + e.Message);
throw;
}
}
// OnDataLossAsync is the restore part of the process. This is NOT an arbitrary name, it is an override of the
// method on StatefulServiceBase.
// The method below looks in the backupFolder directory (where the backup files were
// stored in BackupCallbackAsync above), searches for the folder of backup files that have the newest ListWriteTime, then restores them.
// Once the RestoreDescription object is told where to find your data, RestoreAsync is called
// to restore that data to the reliable objects
// Each partition will have OnDataLossAsync called on it
protected override async Task <bool> OnDataLossAsync(RestoreContext restoreCtx,
CancellationToken cancellationToken)
{
string backupFolder;
ServiceEventSource.Current.ServiceMessage(this, "OnDataLoss Invoked!");
this.SetupBackupManager();
try
{
if (this.backupStorageType == BackupManagerType.None)
{
//since we have no backup configured, we return false to indicate
//that state has not changed. This replica will become the basis
//for future replica builds
return(false);
}
else
{
backupFolder =
await this.backupManager.RestoreLatestBackupToTempLocation(cancellationToken);
}
ServiceEventSource.Current.ServiceMessage(this, "Restoration Folder Path " + backupFolder);
RestoreDescription restoreDescription =
new RestoreDescription(backupFolder, RestorePolicy.Force);
// Restore the backup copy
await restoreCtx.RestoreAsync(restoreDescription, cancellationToken);
ServiceEventSource.Current.ServiceMessage(this, "Restore completed");
// Clean up the local temporary directory (the root directory where all backups were unzipped in to)
try
{
DirectoryInfo tempRestoreDirectory = new DirectoryInfo(backupFolder);
tempRestoreDirectory.Delete(true);
}
catch (System.Exception ddel)
{
ServiceEventSource.Current.ServiceMessage(this, "OnDataLossAsync: Delete of backup folder failed with {0}", ddel.Message);
}
// now that a restore has been performed, reset the full backup flag and incremental
// counter so you start fresh again
// You will need to change this type of logic for your own scenario
takeFullBackup = true;
incrementalCount = 0;
return(true);
}
catch (Exception e)
{
ServiceEventSource.Current.ServiceMessage(this, "Restoration failed: " + "{0} {1}" + e.GetType() + e.Message);
throw;
}
}
public async Task RestoreAsync(
RestoreDescription restoreDescription,
CancellationToken cancellationToken)
{
await this.restoreCtx.RestoreAsync(restoreDescription, cancellationToken);
this.IsRestored = true;
this.BackupFolderPath = restoreDescription.BackupFolderPath;
this.BackupPolicy = restoreDescription.Policy;
}
public async Task Restore(IPersistentDownloader downloader, EventArgs eventArgs)
{
await m_init;
if (eventArgs == EventArgs.Empty)
{
// !Note, we disable the event handler before we trigger an active restore, otherwise
// the active restore event will notify the backup controller, while in fact the restore
// command is issued from the controller, and SF backup manager should do it passively.
Log.WriteLine($"{nameof(ServiceFabricBackupManager)}: initiating a new restore operation.");
m_svc.RequestRestore -= OnServiceFabricRequestRestore;
await m_svc.FabricClient.TestManager.StartPartitionDataLossAsync(Guid.NewGuid(), PartitionSelector.PartitionIdOf(m_svc.Context.ServiceName, m_svc.Context.PartitionId), DataLossMode.PartialDataLoss);
await m_sem.WaitAsync();
m_svc.RequestRestore += OnServiceFabricRequestRestore;
return;
}
if (!(eventArgs is RestoreEventArgs rstArgs))
{
throw new NotSupportedException();
}
try
{
var ctx = rstArgs.m_rctx;
var dir = Path.Combine(TrinityConfig.StorageRoot, Path.GetRandomFileName());
var dsc = new RestoreDescription(dir);
var fname = Path.Combine(TrinityConfig.StorageRoot, Path.GetRandomFileName());
using (var file = File.OpenWrite(fname))
{
Log.WriteLine($"{nameof(ServiceFabricBackupManager)}: Downloading ServiceFabric backup data.");
await downloader.DownloadMetadataAsync(MetadataKey, file);
}
Log.WriteLine($"{nameof(ServiceFabricBackupManager)}: Decompressing ServiceFabric backup data.");
FileUtility.CompletePath(dir, create_nonexistent: true);
ZipFile.ExtractToDirectory(fname, dir);
File.Delete(fname);
Log.WriteLine($"{nameof(ServiceFabricBackupManager)}: Restoring ServiceFabric backup data.");
await ctx.RestoreAsync(dsc);
Directory.Delete(dir, recursive: true);
Log.WriteLine($"{nameof(ServiceFabricBackupManager)}: Restored ServiceFabric backup data.");
rstArgs.Complete();
}
catch (Exception ex)
{
rstArgs.Complete(ex);
throw;
}
m_sem.Release();
}
//Dataloss testing can be triggered via powershell. To do so, run the following commands as a script
//Connect-ServiceFabricCluster
//$s = "fabric:/WebReferenceApplication/InventoryService"
//$p = Get-ServiceFabricApplication | Get-ServiceFabricService -ServiceName $s | Get-ServiceFabricPartition | Select -First 1
//$p | Invoke-ServiceFabricPartitionDataLoss -DataLossMode FullDataLoss -ServiceName $s
protected override async Task <bool> OnDataLossAsync(RestoreContext restoreCtx, CancellationToken cancellationToken)
{
ServiceEventSource.Current.ServiceMessage(this, "OnDataLoss Invoked!");
this.SetupBackupManager();
try
{
string backupFolder;
if (this.backupStorageType == BackupManagerType.None)
{
//since we have no backup configured, we return false to indicate
//that state has not changed. This replica will become the basis
//for future replica builds
return(false);
}
else
{
backupFolder = await this.backupManager.RestoreLatestBackupToTempLocation(cancellationToken);
}
ServiceEventSource.Current.ServiceMessage(this, "Restoration Folder Path " + backupFolder);
RestoreDescription restoreRescription = new RestoreDescription(backupFolder, RestorePolicy.Force);
await restoreCtx.RestoreAsync(restoreRescription, cancellationToken);
ServiceEventSource.Current.ServiceMessage(this, "Restore completed");
DirectoryInfo tempRestoreDirectory = new DirectoryInfo(backupFolder);
tempRestoreDirectory.Delete(true);
return(true);
}
catch (Exception e)
{
ServiceEventSource.Current.ServiceMessage(this, "Restoration failed: " + "{0} {1}" + e.GetType() + e.Message);
throw;
}
}
/// <summary>
/// This method is called during suspected data loss.
/// You can use this method to restore the service in case of data loss.
/// </summary>
/// <param name="service"></param>
/// <param name="restoreCtx"></param>
/// <param name="cancellationToken"></param>
/// <returns></returns>
public static async Task <bool> OnDataLossAsync(this IBackupRestoreServiceInternal service, RestoreContext restoreCtx, CancellationToken cancellationToken)
{
//caused by BeginRestoreBackup
service.LogCallback?.Invoke($"BackupRestoreService - OnDataLossAsync starting for partition: {service.Context.PartitionId}.");
var metadata = await service.CentralBackupStore.RetrieveScheduledBackupAsync(service.Context.PartitionId);
if (metadata == null)
{
return(false);
}
List <BackupMetadata> backupList = await GetBackupMetadataAsync(service, metadata);
if (backupList.Count == 0)
{
throw new InvalidOperationException("Failed to find any backups for this partition.");
}
if (backupList[0].BackupOption != BackupOption.Full)
{
throw new InvalidOperationException("Failed to find any full backups for this partition.");
}
string localBackupFolder = Path.Combine(service.Context.CodePackageActivationContext.WorkDirectory, Guid.NewGuid().ToString("N"));
foreach (var backupMetadata in backupList)
{
string subFolder = Path.Combine(localBackupFolder, backupMetadata.TimeStampUtc.ToString("yyyyMMddhhmmss"));
await service.CentralBackupStore.DownloadBackupFolderAsync(backupMetadata.BackupId, subFolder, cancellationToken);
}
var restoreDescription = new RestoreDescription(localBackupFolder, RestorePolicy.Force);
await restoreCtx.RestoreAsync(restoreDescription, cancellationToken);
Directory.Delete(localBackupFolder, true);
service.LogCallback?.Invoke($"BackupRestoreService - OnDataLossAsync complete for partition {service.Context.PartitionId}.");
return(true);
}
protected override async Task <bool> OnDataLossAsync(RestoreContext restoreCtx, CancellationToken cancellationToken)
{
this.SetupBackupManager();
try
{
string backupFolder;
backupFolder = await this.backupManager.RestoreLatestBackupToTempLocation(cancellationToken);
RestoreDescription restoreRescription = new RestoreDescription(backupFolder, RestorePolicy.Force);
await restoreCtx.RestoreAsync(restoreRescription, cancellationToken);
DirectoryInfo tempRestoreDirectory = new DirectoryInfo(backupFolder);
tempRestoreDirectory.Delete(true);
return(true);
}
catch (Exception e)
{
throw;
}
}
public Task RestoreAsync(
RestoreDescription restoreDescription)
{
return(this.RestoreAsync(restoreDescription, CancellationToken.None));
}
// OnDataLossAsync is the restore part of the process. This is NOT an arbitrary name, it is an override of the
// method on StatefulServiceBase.
// The method below looks in the c:\temp\sfbackup directory (where I stored the backup files in BackupCallbackAsync
// above), searches for the folder of backup files that have the newest ListWriteTime, then restore them.
// Once the RestoreDescription object is told where to find your data, RestoreAsync is called
// to restore that data to the reliable objects
protected override async Task <bool> OnDataLossAsync(RestoreContext restoreCtx,
CancellationToken cancellationToken)
{
ServiceEventSource.Current.ServiceMessage(this, "OnDataLoss Invoked!");
this.SetupBackupManager();
try
{
string restorePath = "";
if (this.backupStorageType == BackupManagerType.None)
{
//since we have no backup configured, we return false to indicate
//that state has not changed. This replica will become the basis
//for future replica builds
return(false);
}
else
{
string zipPath = "";
string partitionId = this.Context.PartitionId.ToString("N");
//this will be the root directory for all unzipped files
ICodePackageActivationContext codePackageContext = this.Context.CodePackageActivationContext;
//this.PartitionTempDirectory = Path.Combine(codePackageContext.TempDirectory, partitionId);
this.PartitionTempDirectory = Path.Combine(@"C:\myrestore", partitionId);
// Get back the full list of blobs and do not sort them
// Sorting will be used when restoring a full-only backup
IEnumerable <IListBlobItem> blobs = await this.backupManager.GetBackupBlobs(false);
// the partition temp directory is the root restore directory + partitionId
// create this directory
System.IO.Directory.CreateDirectory(this.PartitionTempDirectory);
foreach (CloudBlockBlob backup in blobs)
{
// this is the full zip path including file name and extension
// this is the location the zip file should be downloaded to
zipPath = Path.Combine(this.PartitionTempDirectory, backup.Name);
// download the zip file from storage
try
{
// download the file in to the partition directory
backup.DownloadToFile(zipPath, FileMode.CreateNew);
}
catch (System.Exception ee)
{
}
string restoreDir = (backup.Name).Remove(backup.Name.Length - 4);
// Create a path in which to unzip the zip file that has been downloaded
// to the partiton directory
//restorePath = Path.Combine(this.PartitionTempDirectory, partitionId);
restorePath = Path.Combine(this.PartitionTempDirectory, restoreDir);
//extract the zip in to the local temp path directory
ZipFile.ExtractToDirectory(zipPath, restorePath);
}
//clean up the zip file, files are now in the restore path
FileInfo zipInfo = new FileInfo(zipPath);
zipInfo.Delete();
//backupFolder =
// await this.backupManager.RestoreLatestBackupToTempLocationIncremental(cancellationToken);
}
ServiceEventSource.Current.ServiceMessage(this, "Restoration Folder Path " + restorePath);
RestoreDescription restoreDescription =
new RestoreDescription(this.PartitionTempDirectory, RestorePolicy.Force);
// Restore the backup copy
await restoreCtx.RestoreAsync(restoreDescription, cancellationToken);
ServiceEventSource.Current.ServiceMessage(this, "Restore completed");
// Clean up the local temporary directory
DirectoryInfo tempRestoreDirectory = new DirectoryInfo(this.PartitionTempDirectory);
tempRestoreDirectory.Delete(true);
return(true);
}
catch (Exception e)
{
ServiceEventSource.Current.ServiceMessage(this, "Restoration failed: " + "{0} {1}" + e.GetType() + e.Message);
throw;
}
}
//Dataloss testing can be triggered via powershell. To do so, run the following commands as a script
//Connect-ServiceFabricCluster
//$s = "fabric:/WebReferenceApplication/InventoryService"
//$p = Get-ServiceFabricApplication | Get-ServiceFabricService -ServiceName $s | Get-ServiceFabricPartition | Select -First 1
//$p | Invoke-ServiceFabricPartitionDataLoss -DataLossMode FullDataLoss -ServiceName $s
protected override async Task<bool> OnDataLossAsync(RestoreContext restoreCtx, CancellationToken cancellationToken)
{
ServiceEventSource.Current.ServiceMessage(this, "OnDataLoss Invoked!");
this.SetupBackupManager();
try
{
string backupFolder;
if (this.backupStorageType == BackupManagerType.None)
{
//since we have no backup configured, we return false to indicate
//that state has not changed. This replica will become the basis
//for future replica builds
return false;
}
else
{
backupFolder = await this.backupManager.RestoreLatestBackupToTempLocation(cancellationToken);
}
ServiceEventSource.Current.ServiceMessage(this, "Restoration Folder Path " + backupFolder);
RestoreDescription restoreRescription = new RestoreDescription(backupFolder, RestorePolicy.Force);
await restoreCtx.RestoreAsync(restoreRescription, cancellationToken);
ServiceEventSource.Current.ServiceMessage(this, "Restore completed");
DirectoryInfo tempRestoreDirectory = new DirectoryInfo(backupFolder);
tempRestoreDirectory.Delete(true);
return true;
}
catch (Exception e)
{
ServiceEventSource.Current.ServiceMessage(this, "Restoration failed: " + "{0} {1}" + e.GetType() + e.Message);
throw;
}
}
/// <summary>
/// This method is called during suspected data loss.
/// You can use this method to restore the service in case of data loss.
/// </summary>
/// <param name="service"></param>
/// <param name="restoreCtx"></param>
/// <param name="cancellationToken"></param>
/// <returns></returns>
public static async Task <bool> OnDataLossAsync(this IBackupRestoreServiceOperations service, RestoreContext restoreCtx, CancellationToken cancellationToken)
{
//Ususally caused by BeginRestoreBackup
//First, query the scheduled metadata for this partition backup
service.LogCallback?.Invoke($"{nameof(BackupRestoreServiceOperations)} - {nameof(OnDataLossAsync)} - Starting for partition: {service.Context.PartitionId}.");
BackupMetadata metadata;
try
{
metadata = await service.CentralBackupStore.RetrieveScheduledBackupAsync(service.Context.PartitionId);
}
catch (Exception ex)
{
string message = $"Failed to find backup information for partition {service.Context.PartitionId}";
service.LogCallback?.Invoke($"{nameof(BackupRestoreServiceOperations)} - {nameof(OnDataLossAsync)} - Failed for partition: {service.Context.PartitionId}. Message:{message} - Error: {ex.Message}");
throw new Exception(message, ex);
}
if (metadata == null)
{
return(false);
}
//Get all metadata for all backups related to this one, if it's an incremental one
List <BackupMetadata> backupList;
try
{
backupList = await GetBackupMetadataAsync(service, metadata);
if (backupList.Count < 1)
{
throw new InvalidOperationException("Failed to find any backups for this partition.");
}
if (backupList[0].BackupOption != BackupOption.Full)
{
throw new InvalidOperationException("Failed to find any full backups for this partition.");
}
}
catch (Exception ex)
{
string message = $"Failed to find backup information for partition {service.Context.PartitionId}";
service.LogCallback?.Invoke($"{nameof(BackupRestoreServiceOperations)} - {nameof(OnDataLossAsync)} - Failed for partition: {service.Context.PartitionId}. Message:{message} - Error: {ex.Message}");
throw new Exception(message, ex);
}
//download central to local
string localBackupFolder = Path.Combine(service.Context.CodePackageActivationContext.WorkDirectory, Guid.NewGuid().ToString("N"));
try
{
foreach (var backupMetadata in backupList)
{
string subFolder = Path.Combine(localBackupFolder, backupMetadata.TimeStampUtc.ToString("yyyyMMddhhmmss"));
await service.CentralBackupStore.DownloadBackupFolderAsync(backupMetadata.BackupId, subFolder, cancellationToken);
}
}
catch (Exception ex)
{
string message = $"Failed to download backup data for partition {service.Context.PartitionId}";
service.LogCallback?.Invoke($"{nameof(BackupRestoreServiceOperations)} - {nameof(OnDataLossAsync)} - Failed for partition: {service.Context.PartitionId}. Message:{message} - Error: {ex.Message}");
throw new Exception(message, ex);
}
//restore
try
{
var restoreDescription = new RestoreDescription(localBackupFolder, RestorePolicy.Force);
await restoreCtx.RestoreAsync(restoreDescription, cancellationToken);
}
catch (Exception ex)
{
string message = $"Failed to restore backup for partition {service.Context.PartitionId}";
service.LogCallback?.Invoke($"{nameof(BackupRestoreServiceOperations)} - {nameof(OnDataLossAsync)} - Failed for partition: {service.Context.PartitionId}. Message:{message} - Error: {ex.Message}");
throw new Exception(message, ex);
}
finally
{
Directory.Delete(localBackupFolder, true);
}
//success!!
service.LogCallback?.Invoke($"BackupRestoreService - OnDataLossAsync complete for partition {service.Context.PartitionId}.");
return(true);
}
