/// <summary>
/// This method communicates with policystorage service and gets the backupstorage details
/// </summary>
/// <param name="policy">Policy which is to be restored</param>
/// <returns>BackupStorage details</returns>
public async Task <BackupStorage> GetBackupStorageDetails(string policy)
{
IPolicyStorageService policyStorageClient = ServiceProxy.Create <IPolicyStorageService>(new Uri("fabric:/StandByApplication/PolicyStorageService"));
BackupStorage backupStorage = await policyStorageClient.GetPolicyStorageDetails(policy);
return(backupStorage);
}
public virtual void Setup()
{
_backupProcess = A.Fake<BackupProcess>();
_backupStorage = A.Fake<BackupStorage>();
_backupNotifier = A.Fake<BackupNotifier>();
_backupCleanUp = A.Fake<BackupCleanUp>();
}
private async Task <PolicyStorageEntity> GetPolicyDetails(string httpConnectionString, string thumbprint, string policyName)
{
PolicyStorageEntity policyStorageEntity = new PolicyStorageEntity
{
policy = policyName
};
string URL = httpConnectionString + "/";
string URLParameters = "BackupRestore/BackupPolicies/" + policyName + "?api-version=6.4";
HttpResponseMessage response = await Utility.HTTPGetAsync(URL, URLParameters, thumbprint);
if (response.IsSuccessStatusCode)
{
// Parse the response body. Blocking!
var content = response.Content.ReadAsAsync <JObject>().Result;
JObject objectData = (JObject)content["Storage"];
BackupStorage backupStorage = JsonConvert.DeserializeObject <BackupStorage>(objectData.ToString());
policyStorageEntity.backupStorage = backupStorage;
return(policyStorageEntity);
}
else
{
Console.WriteLine("{0} ({1})", (int)response.StatusCode, response.ReasonPhrase);
return(null);
}
}
// This encapsulates a restore task which triggers the restore and returns the restore result accordingly.
public async Task <RestoreResult> RestoreWorkFlow(JToken latestbackupInfo, string policy, PartitionWrapper partition, String clusterConnectionString, String clusterThumbprint)
{
string URL = clusterConnectionString + "/";
string URLParameters = "Partitions/" + partition.partitionId + "/$/Restore" + "?api-version=6.4";
BackupStorage backupStorage = await GetBackupStorageDetails(policy);
if (backupStorage == null)
{
ServiceEventSource.Current.ServiceMessage(this.Context, "backupstorage is null");
return(null);
}
BackupInfo backupInfo = new BackupInfo(latestbackupInfo["BackupId"].ToString(), latestbackupInfo["BackupLocation"].ToString(), backupStorage, (DateTime)latestbackupInfo["CreationTimeUtc"]);
HttpResponseMessage response = await Utility.HTTPPostAsync(URL, URLParameters, backupInfo, clusterThumbprint);
if (response.IsSuccessStatusCode || response.StatusCode == System.Net.HttpStatusCode.Conflict) // As calling Restore multiple times results in 409/Conflict Error, when in progress
{
string restoreState = "";
restoreState = GetRestoreState(partition, clusterConnectionString, clusterThumbprint);
return(new RestoreResult(backupInfo, restoreState));
}
else
{
Console.WriteLine("{0} ({1})", (int)response.StatusCode, response.ReasonPhrase);
return(null);
}
}
public BackupInfo(string backupId, string backupLocation, BackupStorage backupStorage, DateTime backupTime)
{
this.backupId = backupId;
this.backupLocation = backupLocation;
this.backupStorage = backupStorage;
this.backupTime = backupTime;
}
public GetRestorePointsWithBackupStorageOperationWithinDates(BackupStorage backupStorage, string fabricRequestHeader, DateTime startDateTime, DateTime endDateTime
, string serviceName, string apiVersion, StatefulService statefulService)
: base(fabricRequestHeader, serviceName, apiVersion, statefulService)
{
this.backupStorage = backupStorage;
this.startDateTime = startDateTime;
this.endDateTime = endDateTime;
}
internal BackupPartitionWorkItem(string serviceUri, string partitionId, BackupStorage backupStorage, Guid backupRequestGuid, int backupTimeoutinMinutes)
{
this.ServiceUri = serviceUri;
this.PartitionId = partitionId;
this.BackupRequestGuid = backupRequestGuid;
this.BackupStorage = backupStorage;
this.RequestDateTime = DateTime.UtcNow;
this.BackupTimeout = TimeSpan.FromMinutes(backupTimeoutinMinutes);
}
public BackupController(
BackupProcess backupProcess,
BackupStorage backupStorage,
BackupCleanUp backupCleanUp,
BackupNotifier backupNotifier)
{
_backupProcess = backupProcess;
_backupStorage = backupStorage;
_backupCleanUp = backupCleanUp;
_backupNotifier = backupNotifier;
}
// This encapsulates a restore task which triggers the restore and returns the restore result accordingly.
public async Task <RestoreResult> RestoreWorkFlow(JToken latestbackupInfo, string policy, PartitionWrapper partition, String clusterConnectionString, String clusterThumbprint, String partitionDictionary)
{
IReliableDictionary <Guid, PartitionWrapper> myDictionary = await this.StateManager.GetOrAddAsync <IReliableDictionary <Guid, PartitionWrapper> >(partitionDictionary);
string URL = clusterConnectionString + "/";
string urlParameters = "Partitions/" + partition.partitionId + "/$/Restore" + "?api-version=6.2-preview";
X509Certificate2 clientCert = GetClientCertificate(clusterThumbprint);
WebRequestHandler requestHandler = new WebRequestHandler();
requestHandler.ClientCertificates.Add(clientCert);
requestHandler.ServerCertificateValidationCallback = this.MyRemoteCertificateValidationCallback;
HttpClient client = new HttpClient(requestHandler)
{
BaseAddress = new Uri(URL)
};
client.DefaultRequestHeaders.Accept.Add(
new MediaTypeWithQualityHeaderValue("application/json"));
BackupStorage backupStorage = await GetBackupStorageDetails(policy);
if (backupStorage == null)
{
ServiceEventSource.Current.ServiceMessage(this.Context, "backupstorage is null");
return(null);
}
BackupInfo backupInfo = new BackupInfo(latestbackupInfo["BackupId"].ToString(), latestbackupInfo["BackupLocation"].ToString(), backupStorage, (DateTime)latestbackupInfo["CreationTimeUtc"]);
HttpResponseMessage response = await client.PostAsJsonAsync(urlParameters, backupInfo); // Blocking call!
if (response.IsSuccessStatusCode || response.StatusCode == System.Net.HttpStatusCode.Conflict) // As calling Restore multiple times results in 409/Conflict Error, when in progress
{
string restoreState = "";
//do
//{
// await Task.Delay(10000);
// restoreState = GetRestoreState(partition, clusterConnectionString, clusterThumbprint);
//} while (restoreState.Equals("Accepted") || restoreState.Equals("RestoreInProgress"));
restoreState = GetRestoreState(partition, clusterConnectionString, clusterThumbprint);
return(new RestoreResult(backupInfo, restoreState));
}
else
{
Console.WriteLine("{0} ({1})", (int)response.StatusCode, response.ReasonPhrase);
return(null);
}
}
private BackupStoreInformation GetBackupStoreInformationFromBackupStorage(BackupStorage backupStorage)
{
BackupStoreInformation storeInformation;
switch (backupStorage.BackupStorageType)
{
case BackupStorageType.AzureBlobStore:
var azureStorage = (AzureBlobBackupStorageInfo)backupStorage;
storeInformation = new AzureBlobBackupStore
{
ConnectionString = azureStorage.ConnectionString,
ContainerName = azureStorage.ContainerName,
FolderPath = UtilityHelper.GetBaseDirectoryPathForPartition(this.ServiceUri, this.PartitionId), // TODO: This should be constructed
IsAccountKeyEncrypted = azureStorage.IsConnectionStringEncrypted,
};
break;
case BackupStorageType.DsmsAzureBlobStore:
var dsmsAzureStorage = (DsmsAzureBlobBackupStorageInfo)backupStorage;
storeInformation = new DsmsAzureBlobBackupStore
{
StorageCredentialsSourceLocation = dsmsAzureStorage.StorageCredentialsSourceLocation,
ContainerName = dsmsAzureStorage.ContainerName,
FolderPath = UtilityHelper.GetBaseDirectoryPathForPartition(this.ServiceUri, this.PartitionId),
};
break;
case BackupStorageType.FileShare:
var fileShareStorage = (FileShareBackupStorageInfo)backupStorage;
storeInformation = new FileShareBackupStore
{
AccessType = String.IsNullOrEmpty(fileShareStorage.PrimaryUserName) ? FileShareAccessType.None : FileShareAccessType.DomainUser,
FileSharePath = Path.Combine(fileShareStorage.Path, UtilityHelper.GetBaseDirectoryPathForPartition(this.ServiceUri, this.PartitionId)),
PrimaryUserName = fileShareStorage.PrimaryUserName,
PrimaryPassword = fileShareStorage.PrimaryPassword,
SecondaryUserName = fileShareStorage.SecondaryUserName,
SecondaryPassword = fileShareStorage.SecondaryPassword,
IsPasswordEncrypted = fileShareStorage.IsPasswordEncrypted,
};
break;
default:
throw new ArgumentException(string.Format("{0} not supported.", backupStorage.BackupStorageType));
}
return(storeInformation);
}
string DownloadPackageBackup(
string id,
string version,
string hash)
{
var blobClient = BackupStorage.CreateCloudBlobClient();
var packageBackupsBlobContainer = Util.GetPackageBackupsBlobContainer(blobClient);
var packageBackupFileName = Util.GetPackageBackupFileName(
id,
version,
hash);
var packageBackupBlob = packageBackupsBlobContainer.GetBlockBlobReference(packageBackupFileName);
var downloadPath = Path.Combine(_tempFolder, packageBackupFileName);
packageBackupBlob.DownloadToFile(downloadPath);
return(downloadPath);
}
internal override async Task <HttpResponseMessage> RunAsync(TimeSpan timeout, CancellationToken cancellationToken)
{
string applicationNameUri;
string serviceNameUri;
string partitionId;
string fabricUri = await UtilityHelper.GetFabricUriFromRequstHeaderForPartitions(this.fabricRequestHeader, timeout, cancellationToken);
UtilityHelper.GetApplicationAndServicePartitionUri(fabricUri, out applicationNameUri, out serviceNameUri, out partitionId);
BackupStorage backupStorageModel;
if (this.backupStorage == null)
{
var backupMapping = (await this.BackupMappingStore.GetValueAsync(fabricUri) ??
await this.BackupMappingStore.GetValueAsync(serviceNameUri)) ??
await this.BackupMappingStore.GetValueAsync(UtilityHelper.GetApplicationNameFromService(applicationNameUri));
if (backupMapping == null)
{
throw new FabricPeriodicBackupNotEnabledException();
}
var backupPolicy = await this.BackupPolicyStore.GetValueAsync(backupMapping.BackupPolicyName, timeout, cancellationToken);
backupStorageModel = backupPolicy.Storage;
}
else
{
backupStorageModel = BackupStorage.FromBackupStorageView(this.backupStorage);
}
var backupRequestGuid = Guid.NewGuid();
var backupPartitionStatus = new BackupPartitionStatus(fabricUri, backupRequestGuid);
var sendBackupPartitionToServiceNodeWorkItem = new BackupPartitionWorkItem(serviceNameUri, partitionId, backupStorageModel, backupRequestGuid, this.backupTimeoutinMinutes);
using (var transaction = this.StatefulService.StateManager.CreateTransaction())
{
await this.CheckForEitherBackupOrRestoreInProgress(fabricUri, timeout, cancellationToken, transaction);
await this.WorkItemQueue.AddWorkItem(sendBackupPartitionToServiceNodeWorkItem, timeout, cancellationToken, transaction);
await this.BackupPartitionStore.AddOrUpdateAsync(fabricUri, backupPartitionStatus, (fabricUriKey, backupPartitionStatusUpdate) => backupPartitionStatus, timeout, cancellationToken, transaction);
await transaction.CommitAsync();
}
return(new HttpResponseMessage(HttpStatusCode.Accepted));
}
private async Task <PolicyStorageEntity> getPolicyDetails(string cs, string policyName)
{
PolicyStorageEntity policyStorageEntity = new PolicyStorageEntity();
policyStorageEntity.policy = policyName;
string URL = "https://" + cs + "/";
string urlParameters = "BackupRestore/BackupPolicies/" + policyName + "?api-version=6.2-preview";
X509Certificate2 clientCert = GetClientCertificate();
WebRequestHandler requestHandler = new WebRequestHandler();
requestHandler.ClientCertificates.Add(clientCert);
requestHandler.ServerCertificateValidationCallback = this.MyRemoteCertificateValidationCallback;
HttpClient client = new HttpClient(requestHandler)
{
BaseAddress = new Uri(URL)
};
client.DefaultRequestHeaders.Accept.Add(
new MediaTypeWithQualityHeaderValue("application/json"));
// List data response.
HttpResponseMessage response = await client.GetAsync(urlParameters); // Blocking call!
if (response.IsSuccessStatusCode)
{
// Parse the response body. Blocking!
var content = response.Content.ReadAsAsync <JObject>().Result;
JObject objectData = (JObject)content["Storage"];
BackupStorage backupStorage = JsonConvert.DeserializeObject <BackupStorage>(objectData.ToString());
policyStorageEntity.backupStorage = backupStorage;
return(policyStorageEntity);
}
else
{
Console.WriteLine("{0} ({1})", (int)response.StatusCode, response.ReasonPhrase);
return(null);
}
}
// This encapsulates a restore task which triggers the restore and returns the restore result accordingly.
public async Task <RestoreResult> RestoreWorkFlow(JToken latestbackupInfo, string policy, PartitionWrapper partition, String clusterConnectionString, String partitionDictionary)
{
IReliableDictionary <Guid, PartitionWrapper> myDictionary = await this.StateManager.GetOrAddAsync <IReliableDictionary <Guid, PartitionWrapper> >(partitionDictionary);
HttpClient client = new HttpClient();
string URL = clusterConnectionString + "/Partitions/" + partition.partitionId + "/$/Restore";
string urlParameters = "?api-version=6.2-preview";
client.BaseAddress = new Uri(URL);
client.DefaultRequestHeaders.Accept.Add(
new MediaTypeWithQualityHeaderValue("application/json"));
BackupStorage backupStorage = await GetBackupStorageDetails(policy);
if (backupStorage == null)
{
ServiceEventSource.Current.ServiceMessage(this.Context, "backupstorage is null");
return(null);
}
BackupInfo backupInfo = new BackupInfo(latestbackupInfo["BackupId"].ToString(), latestbackupInfo["BackupLocation"].ToString(), backupStorage, (DateTime)latestbackupInfo["CreationTimeUtc"]);
HttpResponseMessage response = await client.PostAsJsonAsync(urlParameters, backupInfo); // Blocking call!
if (response.IsSuccessStatusCode)
{
string restoreState = "";
do
{
await Task.Delay(10000);
restoreState = GetRestoreState(partition, clusterConnectionString);
} while (restoreState.Equals("Accepted") || restoreState.Equals("RestoreInProgress"));
return(new RestoreResult(backupInfo, restoreState));
}
else
{
Console.WriteLine("{0} ({1})", (int)response.StatusCode, response.ReasonPhrase);
return(null);
}
}
public async Task <IActionResult> GetPolicies(String cs, [FromBody] List <string> applications)
{
List <PolicyStorageEntity> policyDetails = new List <PolicyStorageEntity>();
List <string> policyNames = new List <string>();
foreach (string application in applications)
{
string URL = "http://" + cs + "/Applications/" + application + "/$/GetBackupConfigurationInfo";
string urlParameters = "?api-version=6.2-preview";
HttpClient client = new HttpClient
{
BaseAddress = new Uri(URL)
};
client.DefaultRequestHeaders.Accept.Add(
new MediaTypeWithQualityHeaderValue("application/json"));
HttpResponseMessage response = await client.GetAsync(urlParameters);
if (response.IsSuccessStatusCode)
{
var content = response.Content.ReadAsAsync <JObject>().Result;
JArray array = (JArray)content["Items"];
foreach (var item in array)
{
string policy = item["PolicyName"].ToString();
if (!policyNames.Contains(policy))
{
policyNames.Add(policy);
}
}
}
else
{
Console.WriteLine("{0} ({1})", (int)response.StatusCode, response.ReasonPhrase);
return(null);
}
foreach (string policyName in policyNames)
{
PolicyStorageEntity policyStorageEntity = new PolicyStorageEntity();
policyStorageEntity.policy = policyName;
URL = "http://" + cs + "/BackupRestore/BackupPolicies/" + policyName;
urlParameters = "?api-version=6.2-preview";
client = new HttpClient
{
BaseAddress = new Uri(URL)
};
client.DefaultRequestHeaders.Accept.Add(
new MediaTypeWithQualityHeaderValue("application/json"));
// List data response.
response = await client.GetAsync(urlParameters); // Blocking call!
if (response.IsSuccessStatusCode)
{
// Parse the response body. Blocking!
var content = response.Content.ReadAsAsync <JObject>().Result;
JObject objectData = (JObject)content["Storage"];
BackupStorage backupStorage = JsonConvert.DeserializeObject <BackupStorage>(objectData.ToString());
policyStorageEntity.backupStorage = backupStorage;
policyDetails.Add(policyStorageEntity);
}
else
{
Console.WriteLine("{0} ({1})", (int)response.StatusCode, response.ReasonPhrase);
return(null);
}
}
}
return(this.Json(policyDetails));
}
private BackupStorage UpdateSecretInStorage(BackupStorage storage, string thumbprintFromManifest, string x509StoreNameFromManifest, CancellationToken cancellationToken)
{
if (storage.BackupStorageType == BackupStorageType.AzureBlobStore)
{
var azureStorage = (AzureBlobBackupStorageInfo)storage;
if (!azureStorage.IsConnectionStringEncrypted && String.IsNullOrEmpty(thumbprintFromManifest))
{
return(storage);
}
if (azureStorage.IsConnectionStringEncrypted)
{
return(azureStorage.ToBuilder()
.WithConnectionString(UtilityHelper.ConvertToUnsecureString(EncryptionUtility.DecryptText(azureStorage.ConnectionString)))
.SetIsConnectionStringEncrypted(false)
.Build());
}
if (!String.IsNullOrEmpty(thumbprintFromManifest))
{
return(azureStorage.ToBuilder()
.WithConnectionString(EncryptionUtility.EncryptText(azureStorage.ConnectionString, thumbprintFromManifest, x509StoreNameFromManifest))
.SetIsConnectionStringEncrypted(true)
.Build());
}
}
else if (storage.BackupStorageType == BackupStorageType.FileShare)
{
var fileShareStorage = (FileShareBackupStorageInfo)storage;
if (!fileShareStorage.IsPasswordEncrypted && String.IsNullOrEmpty(thumbprintFromManifest))
{
return(storage);
}
if (fileShareStorage.IsPasswordEncrypted)
{
var builder = fileShareStorage.ToBuilder()
.SetIsPasswordEncrypted(false);
if (!String.IsNullOrEmpty(fileShareStorage.PrimaryPassword))
{
builder.WithPrimaryPassword(UtilityHelper.ConvertToUnsecureString(EncryptionUtility.DecryptText(fileShareStorage.PrimaryPassword)));
}
if (!String.IsNullOrEmpty(fileShareStorage.SecondaryPassword))
{
builder.WithSecondaryPassword(UtilityHelper.ConvertToUnsecureString(EncryptionUtility.DecryptText(fileShareStorage.SecondaryPassword)));
}
return(builder.Build());
}
if (!String.IsNullOrEmpty(thumbprintFromManifest))
{
var builder = fileShareStorage.ToBuilder()
.SetIsPasswordEncrypted(true);
if (!String.IsNullOrEmpty(fileShareStorage.PrimaryPassword))
{
builder.WithPrimaryPassword(EncryptionUtility.EncryptText(fileShareStorage.PrimaryPassword, thumbprintFromManifest, x509StoreNameFromManifest));
}
if (!String.IsNullOrEmpty(fileShareStorage.SecondaryPassword))
{
builder.WithSecondaryPassword(EncryptionUtility.EncryptText(fileShareStorage.SecondaryPassword, thumbprintFromManifest, x509StoreNameFromManifest));
}
return(builder.Build());
}
}
else if (storage.BackupStorageType == BackupStorageType.DsmsAzureBlobStore)
{
// DsmsAzureBlobStorage doesn't contain any encrypted strings, return the original value.
return(storage);
}
throw new ArgumentException("Invalid storage type");
}
public override void ExecuteCommand()
{
Log.Info(
"Backing up '{0}/packages' -> '{1}/package-backups'.",
StorageAccount.Credentials.AccountName,
BackupStorage.Credentials.AccountName);
var client = CreateBlobClient();
var backupClient = BackupStorage.CreateCloudBlobClient();
// Get the state file object
var state = GetStateFile(backupClient);
var lastId = state.LastBackedUpId;
bool forcedRecheck = false;
if (state.LastBackupCompletedUtc.HasValue && ((DateTimeOffset.UtcNow - state.LastBackupCompletedUtc.Value) > TimeSpan.FromDays(1)))
{
// Do a "full" backup (check every package file) every day
lastId = null;
forcedRecheck = true;
}
var packagesToBackUp = GetPackagesToBackUp(lastId, forcedRecheck);
var processedCount = 0;
var backupBlobs = backupClient.GetContainerReference("package-backups");
var packageBlobs = client.GetContainerReference("packages");
if (!WhatIf)
{
backupBlobs.CreateIfNotExists();
}
Parallel.ForEach(packagesToBackUp, new ParallelOptions {
MaxDegreeOfParallelism = SingleThreaded ? 1 : 10
}, package =>
{
try
{
var packageBlob = packageBlobs.GetBlockBlobReference(Util.GetPackageFileName(package.Id, package.Version));
var backupBlob = backupBlobs.GetBlockBlobReference(Util.GetPackageBackupFileName(package.Id, package.Version, package.Hash));
if (packageBlob.Exists())
{
bool shouldCopy = backupBlob.Exists();
// Verify the package, if it exists
if (shouldCopy)
{
packageBlob.FetchAttributes();
backupBlob.FetchAttributes();
shouldCopy =
String.IsNullOrEmpty(packageBlob.Properties.ContentMD5) ||
String.IsNullOrEmpty(backupBlob.Properties.ContentMD5) ||
!String.Equals(packageBlob.Properties.ContentMD5, backupBlob.Properties.ContentMD5, StringComparison.Ordinal);
}
if (!shouldCopy && !WhatIf)
{
backupBlob.StartCopyFromBlob(packageBlob);
}
Interlocked.Increment(ref processedCount);
Log.Trace(
"[{2:000000}/{3:000000} {4:00.0}%] {5} Backup of '{0}@{1}'.",
package.Id,
package.Version,
processedCount,
packagesToBackUp.Count,
(double)processedCount / (double)packagesToBackUp.Count,
shouldCopy ? "Skipped" : "Started");
}
else
{
Log.Warn(
"[{2:000000}/{3:000000} {4:00.0}%] Package File not found in source: '{0}@{1}'",
package.Id,
package.Version,
processedCount,
packagesToBackUp.Count,
(double)processedCount / (double)packagesToBackUp.Count);
}
}
catch (Exception ex)
{
Interlocked.Increment(ref processedCount);
Log.Error(
"[{2:000000}/{3:000000} {4:00.0}%] Error Starting Backup of '{0}@{1}': {5}",
package.Id,
package.Version,
processedCount,
packagesToBackUp.Count,
(double)processedCount / (double)packagesToBackUp.Count,
ex.Message);
}
});
Log.Info("Backed up {0} packages from {1} to {2}", processedCount, StorageAccount.Credentials.AccountName, BackupStorage.Credentials.AccountName);
state.LastBackupCompletedUtc = DateTimeOffset.UtcNow;
state.LastBackedUpId = packagesToBackUp.Max(p => p.Key);
WriteStateFile(backupClient, state);
}
/// <summary>
/// Gets invoked when the RetentionScheduler's timer's time is elapsed
/// </summary>
internal void timerCallback(string backupPolicyName)
{
RetentionScheduler retentionScheduler;
if (!RetentionSchedulerDict.TryGetValue(backupPolicyName, out retentionScheduler))
{
BackupRestoreTrace.TraceSource.WriteWarning(TraceType, "RetentionScheduler object is not found in retentionScheduler dict for policy: {0}", backupPolicyName);
return;
}
bool toRearm = true;
RetentionMetadata retentionMetadata = this.RetentionStore.GetValueAsync(backupPolicyName, this.timeout, this.tokenSource.Token).GetAwaiter().GetResult();
try
{
BackupRestoreTrace.TraceSource.WriteInfo(TraceType, " TimerCallback running for backupPolicy : {0}", backupPolicyName);
var backupPolicyStore = BackupPolicyStore.CreateOrGetBackupPolicyStore(this.StatefulService).GetAwaiter().GetResult();
BackupPolicy backupPolicy = backupPolicyStore.GetValueAsync(backupPolicyName, this.timeout, this.tokenSource.Token).GetAwaiter().GetResult();
if (backupPolicy == null)
{
/**
* This is used to garbage collect the retention scheduler. It could happen while deleting\updating policy retention scheduler object was not
* stopped properly. So, timer will call its callback again just to find that the backup policy does not exists.
* In this case, we should stop the retentionScheduler.
* **/
RetentionScheduler retentionSchedulerToDelete;
if (RetentionSchedulerDict.TryRemove(backupPolicyName, out retentionSchedulerToDelete))
{
retentionSchedulerToDelete.Stop();
}
else
{
BackupRestoreTrace.TraceSource.WriteInfo(TraceType, "TimerCallback TryRemove retentionSchedulerDict failed of policy {0}. It could happen" +
" when residual callback already scheduled on threadpool thread is getting invoked.", backupPolicyName);
}
toRearm = false;
return;
}
HashSet <string> partitionsEnabledByPolicy = FindParititionsEnabledByPolicy(backupPolicy).GetAwaiter().GetResult();
retentionMetadata.UpdateLastRetentionStartTime();
this.RetentionStore.UpdateValueAsync(backupPolicyName, retentionMetadata, this.timeout, this.tokenSource.Token, null).GetAwaiter().GetResult();
var derivedRetentionPolicy = (BasicRetentionPolicy)retentionMetadata.CurrentRetentionPolicy;
TimeSpan retentionDuration = derivedRetentionPolicy.RetentionDuration;
int minimumNumberOfBackups = derivedRetentionPolicy.MinimumNumberOfBackups;
foreach (var partitionId in partitionsEnabledByPolicy)
{
BackupRestoreTrace.TraceSource.WriteInfo(TraceType, " Deleting backups for partitionId : {0}", partitionId);
List <string> partitionList = this.CleanupStore.GetValueAsync(backupPolicyName, this.timeout, this.tokenSource.Token).GetAwaiter().GetResult();
if (!(partitionList == null || partitionList.Count == 0) && partitionList.Contains(partitionId))
{
// Cleanup is going on for this partition.
continue;
}
DateTime endDateFilter = DateTime.UtcNow - retentionDuration;
BackupStorage storage = backupPolicy.Storage;
DeleteFilesForPartition(partitionId, endDateFilter, minimumNumberOfBackups, storage, backupPolicyName).GetAwaiter().GetResult();
this.tokenSource.Token.ThrowIfCancellationRequested();
}
// --> Introduce some wait time here so that CompletionTime is never equal to StartTime.
retentionMetadata.UpdateLastRetentionCompletionTime();
this.RetentionStore.UpdateValueAsync(backupPolicyName, retentionMetadata).GetAwaiter().GetResult();
retentionScheduler.ArmTimer(retentionMetadata);
toRearm = false;
}
catch (Exception ex)
{
if (ex is OperationCanceledException)
{
// Since, timercallback was cancelled, therefore, there is no need to rearm the timer.
BackupRestoreTrace.TraceSource.WriteWarning(TraceType, " TimerCallback for backupPolicy : {0} was cancelled ", backupPolicyName);
toRearm = false;
}
else
{
BackupRestoreTrace.TraceSource.WriteWarning(TraceType, " TimerCallback for backupPolicy : {0} has thrown an exception : {1}", backupPolicyName, ex);
}
}
finally
{
if (toRearm)
{
retentionScheduler.RearmTimer(true, retentionMetadata);
}
}
}
private async Task DeleteFilesForPartition(string partitionId, DateTime endDateFilter, int minimumNumberOfBackups, BackupStorage storage, string backupPolicyName, bool isCleanupTask = false)
{
// Design:
/**
* Need out Two API's from Azure share and file share
* 1. List of all backups before a certain date for the partition
* 2. Start deleting the files one by one.(one file at a time with some random wait time.)
*
**/
BackupRestoreTrace.TraceSource.WriteInfo(TraceType, " In DeleteFilesForPartition for partitionId : {0} with endDateFilter {1}", partitionId, endDateFilter);
string applicationName = null;
string serviceName = null;
string partitionID = null;
string fabricUri = await UtilityHelper.GetFabricUriFromPartitionId(partitionId, this.timeout, this.tokenSource.Token);
var FabricBackupResourceType = UtilityHelper.GetApplicationAndServicePartitionName(fabricUri, out applicationName,
out serviceName, out partitionID);
Debug.Assert(partitionId == partitionID);
var storeManager = RecoveryPointManagerFactory.GetRecoveryPointManager(storage);
this.tokenSource.Token.ThrowIfCancellationRequested();
BRSContinuationToken bRSContinuationToken = new BRSContinuationToken();
List <string> allBackupFiles = storeManager.EnumerateRecoveryPointMetadataFiles(storage.GetRelativeStorePath(applicationName, serviceName, partitionId),
DateTime.MinValue, DateTime.MaxValue, false, bRSContinuationToken, 0);
List <RestorePoint> restorePoints = new List <RestorePoint>();
if ((allBackupFiles != null) && (allBackupFiles.Count > 0))
{
restorePoints = await storeManager.GetRecoveryPointDetailsAsync(allBackupFiles, this.tokenSource.Token);
}
this.tokenSource.Token.ThrowIfCancellationRequested();
int backupCount = allBackupFiles.Count;
if (backupCount <= minimumNumberOfBackups)
{
return;
}
restorePoints = restorePoints.OrderBy(restorePoint => restorePoint.CreationTimeUtc).ToList();
restorePoints = restorePoints.Where(restorePoint => restorePoint.CreationTimeUtc < endDateFilter).ToList();
restorePoints.Reverse();
bool lastFullBackupFound = false;
RetentionMetadata retentionMetadata = await this.RetentionStore.GetValueAsync(backupPolicyName, this.timeout, this.tokenSource.Token);
if (isCleanupTask)
{
lastFullBackupFound = true;
}
if (!isCleanupTask && retentionMetadata.OnGoingRetentionFile.ContainsKey(partitionId) &&
retentionMetadata.OnGoingRetentionFile[partitionId] != null)
{
storeManager.DeleteBackupFiles(retentionMetadata.OnGoingRetentionFile[partitionId]);
}
foreach (var restorePoint in restorePoints)
{
this.tokenSource.Token.ThrowIfCancellationRequested();
BackupRestoreTrace.TraceSource.WriteInfo(TraceType, " Trying to Delete Restore point with backupLocation: {0}", restorePoint.BackupLocation);
if (!isCleanupTask)
{
BackupRestoreTrace.TraceSource.WriteInfo(TraceType, " Updating OnGoing RetentionFile in the metadata for backupPolicy : {0} with partitionId : {1} and BackupLocation {2}"
, backupPolicyName, partitionId, restorePoint.BackupLocation);
retentionMetadata.UpdateOnGoingRetentionFile(partitionId, restorePoint.BackupLocation);
await this.RetentionStore.UpdateValueAsync(backupPolicyName, retentionMetadata, this.timeout, this.tokenSource.Token, null);
}
if (!lastFullBackupFound)
{
BackupRestoreTrace.TraceSource.WriteInfo(TraceType, " Skipping RestorePoint with location : {0} as full backup is not found", restorePoint.BackupLocation);
if (restorePoint.BackupType == BackupOptionType.Full)
{
BackupRestoreTrace.TraceSource.WriteInfo(TraceType, " Full Backup found so, after restore points after :{0} will be deleted. ", restorePoint.BackupLocation);
lastFullBackupFound = true;
}
}
else
{
if (backupCount <= minimumNumberOfBackups)
{
break;
}
if (storeManager.DeleteBackupFiles(restorePoint.BackupLocation))
{
BackupRestoreTrace.TraceSource.WriteInfo(TraceType, " Deleted Restore point with backupLocation: {0}", restorePoint.BackupLocation);
backupCount--;
}
else
{
BackupRestoreTrace.TraceSource.WriteWarning(TraceType, " Not able to delete Restore point with backupLocation: {0}", restorePoint.BackupLocation);
}
}
await IntroduceRandomDelay();
}
// Deletion for a partition is completed.
if (!isCleanupTask)
{
BackupRestoreTrace.TraceSource.WriteInfo(TraceType, " Updating OnGoing RetentionFile in the metadata for backupPolicy : {0}", backupPolicyName);
retentionMetadata.UpdateOnGoingRetentionFile(partitionId);
await this.RetentionStore.UpdateValueAsync(backupPolicyName, retentionMetadata, this.timeout, this.tokenSource.Token, null);
}
}
