public static Task DownloadToStreamAsync(CloudBlob blob, Stream target, CancellationToken cancellationToken)
{
try
{
return(BackupRestoreUtility.PerformActionWithRetriesAsync <CloudBlob, Stream>(
(CloudBlob blockBlob, Stream targetStream, CancellationToken ct) =>
{
return blockBlob.DownloadToStreamAsync(targetStream, cancellationToken);
},
blob,
target,
cancellationToken,
new RetriableOperationExceptionHandler(AzureStorageExceptionHandler)));
}
catch (Exception e)
{
TraceSource.WriteExceptionAsError(
TraceType,
e,
"Failed to download blob {0} to stream.",
blob.Name);
throw;
}
}
public static CloudBlobContainer GetContainer(string connectionString, string containerName)
{
try
{
return(BackupRestoreUtility.PerformWithRetries <string, CloudBlobContainer>(
(string name) =>
{
CloudBlobContainer container = GetContainerRef(connectionString, name);
// Create the container
if (!container.Exists())
{
throw new ArgumentException(string.Format("Given container {0} doesn't exist.", name), "containerName");
}
return container;
},
containerName,
new RetriableOperationExceptionHandler(AzureStorageExceptionHandler),
RetryCount));
}
catch (Exception e)
{
TraceSource.WriteExceptionAsError(
TraceType,
e,
"Failed to get container {0} from Azure storage.",
containerName);
throw;
}
}
public List <string> EnumerateSubFolders(string relativePath)
{
if (String.IsNullOrEmpty(relativePath))
{
throw new ArgumentException();
}
var remotePath = Path.Combine(this.storeInformation.Path, relativePath);
try
{
var enumerateInfo = new FolderEnumerateInfo
{
DirectoryPath = remotePath,
GetFullPath = false
};
BackupRestoreUtility.PerformIOWithRetries(
enumInfo =>
{
PerformDestinationOperation(EnumerateFolders, enumInfo);
},
enumerateInfo);
return(enumerateInfo.SubFolders.ToList());
}
catch (Exception e)
{
throw new IOException(String.Format("Unable to enumerate sub folders from location {0}.", remotePath), e);
}
}
public async Task <List <string> > GetBackupLocationsInBackupChainAsync(string backupLocation, CancellationToken cancellationToken)
{
try
{
return(await BackupRestoreUtility.PerformIOWithRetriesAsync <string, List <string> >(
GetBackupLocationsInBackupChainInternalAsync,
backupLocation,
cancellationToken));
}
catch (Exception e)
{
throw new IOException("Unable to get backup locations in chain.", e);
}
}
public async Task <List <RestorePoint> > GetRecoveryPointDetailsAsync(IEnumerable <string> recoveryPointMetadataFiles, CancellationToken cancellationToken)
{
try
{
return(await BackupRestoreUtility.PerformIOWithRetriesAsync <IEnumerable <string>, List <RestorePoint> >(
GetRecoveryPointDetailsInternalAsync,
recoveryPointMetadataFiles,
cancellationToken));
}
catch (Exception e)
{
throw new IOException("Unable to get recovery point details.", e);
}
}
internal async Task CleanUpTaskUtil(string partitionId, BackupPolicy backupPolicy)
{
try
{
BackupRestoreTrace.TraceSource.WriteInfo(TraceType, " Running CleanUp task for partitionId : {0} and backupPolicy: {1}", partitionId, backupPolicy.Name);
await IntroduceRandomDelay();
RetentionMetadata retentionMetadata = await this.RetentionStore.GetValueAsync(backupPolicy.Name, this.timeout, this.tokenSource.Token);
Func <bool> condition = () =>
{
if (retentionMetadata == null || retentionMetadata.LastRetentionStartTime == DateTime.MinValue || retentionMetadata.LastRetentionCompletionTime > retentionMetadata.LastRetentionStartTime ||
!retentionMetadata.OnGoingRetentionFile.ContainsKey(partitionId))
{
return(true);
}
return(false);
};
bool cleanUpComplete = await BackupRestoreUtility.TryPerformOperationWithWaitAsync(
(partition, token) =>
{
return(CleanUpTask(partition, backupPolicy, token));
}, condition, partitionId,
this.tokenSource.Token,
minimumWaitTimeForRetryOperationInMs,
maxRetryCountForCleanup,
maximumWaitTimeForCleanUpRetryOperationInMs
);
if (!cleanUpComplete)
{
BackupRestoreTrace.TraceSource.WriteWarning(TraceType, " CleanUp task for partitionId : {0} and backupPolicy : {1} was not completed." +
"It could be because retention of the partition was going on from a long time which did not complete 40 seconds.", partitionId, backupPolicy.Name);
}
}
catch (Exception ex)
{
if (ex is OperationCanceledException)
{
BackupRestoreTrace.TraceSource.WriteWarning(TraceType, " CleanUp task for partitionId : {0} and backupPolicy : {1} is canceled", partitionId, backupPolicy.Name);
}
else
{
BackupRestoreTrace.TraceSource.WriteError(TraceType, " CleanUp task for partitionId : {0} and backupPolicy : {1} has thrown an exception : {2}", partitionId, backupPolicy.Name, ex);
}
}
}
public IEnumerable <string> ListFolders(string directory, string directoryContinuationToken = null)
{
var directoryEnumerateInfo = new FolderEnumerateInfo()
{
DirectoryPath = directory,
DirectoryPathContinuationToken = directoryContinuationToken,
GetFullPath = true
};
BackupRestoreUtility.PerformIOWithRetries(
enumInfo =>
{
PerformDestinationOperation(EnumerateFolders, enumInfo);
},
directoryEnumerateInfo);
return(directoryEnumerateInfo.SubFolders);
}
private void DeleteFolderOrFile(string relativefilePathOrSourceFolderPath, bool isFolder)
{
var remotePath = Path.Combine(this.storeInformation.Path, relativefilePathOrSourceFolderPath);
if (!CheckIfBackupExists(remotePath))
{
BackupRestoreTrace.TraceSource.WriteWarning("FileShareRecoveryPointManager", "relativefilePathOrSourceFolderPath: {0} with storePath : {1} not found"
, relativefilePathOrSourceFolderPath, this.storeInformation.Path);
return;
}
if (isFolder)
{
var enumerateInfo = new FileEnumerateInfo
{
DirectoryPath = remotePath,
SearchOptionEnum = SearchOption.AllDirectories
};
BackupRestoreUtility.PerformIOWithRetries(
enumInfo =>
{
PerformDestinationOperation(EnumerateFiles, enumInfo);
},
enumerateInfo);
List <string> filePathList = enumerateInfo.Files.ToList();
foreach (var filePath in filePathList)
{
BackupRestoreUtility.PerformIOWithRetries(
file =>
{
PerformDestinationOperation(DeleteFile, file);
},
filePath);
}
}
else
{
BackupRestoreUtility.PerformIOWithRetries(
file =>
{
PerformDestinationOperation(DeleteFile, file);
},
remotePath);
}
}
public IEnumerable <string> ListMetadataFilesInTheFolder(string directoryName)
{
var enumerateInfo = new FileEnumerateInfo
{
DirectoryPath = directoryName,
FileExtension = "*" + RecoveryPointMetadataFileExtension,
SearchOptionEnum = SearchOption.TopDirectoryOnly
};
BackupRestoreUtility.PerformIOWithRetries(
enumInfo =>
{
PerformDestinationOperation(EnumerateFiles, enumInfo);
},
enumerateInfo);
return(enumerateInfo.Files);
}
public static void DeleteBlob(CloudBlockBlob cloudBlockBlob)
{
try
{
BackupRestoreUtility.PerformIOWithRetries(
() =>
{
cloudBlockBlob.DeleteIfExists();
},
RetryCount);
}
catch (Exception e)
{
TraceSource.WriteExceptionAsError(
TraceType,
e,
"Failed to list blobs under directory {0} .",
cloudBlockBlob.Name);
throw;
}
}
public static BlobResultSegment GetBlobList(CloudBlobDirectory directory, bool useFlatBlobListing, BlobContinuationToken blobContinuationToken, int maxResults)
{
try
{
return(BackupRestoreUtility.PerformWithRetries <bool, BlobResultSegment>(
(bool flatBlobListing) =>
{
return directory.ListBlobsSegmented(flatBlobListing, BlobListingDetails.None, maxResults, blobContinuationToken, null, null);
},
useFlatBlobListing,
new RetriableOperationExceptionHandler(AzureStorageExceptionHandler),
RetryCount));
}
catch (Exception e)
{
TraceSource.WriteExceptionAsError(
TraceType,
e,
"Failed to list blobs under directory {0} from Azure storage.",
directory.Prefix);
throw;
}
}
public static IEnumerable <IListBlobItem> GetBlobList(CloudBlobDirectory directory, bool useFlatBlobListing)
{
try
{
return(BackupRestoreUtility.PerformWithRetries <bool, IEnumerable <IListBlobItem> >(
(bool flatBlobListing) =>
{
return directory.ListBlobs(flatBlobListing);
},
useFlatBlobListing,
new RetriableOperationExceptionHandler(AzureStorageExceptionHandler),
RetryCount));
}
catch (Exception e)
{
TraceSource.WriteExceptionAsError(
TraceType,
e,
"Failed to list blobs under directory {0} from Azure storage.",
directory.Prefix);
throw;
}
}
public static CloudBlockBlob GetCloudBlockBlobRef(CloudBlobContainer container, string blockBlobPath)
{
try
{
return(BackupRestoreUtility.PerformWithRetries <string, CloudBlockBlob>(
(string blobPath) =>
{
return container.GetBlockBlobReference(blobPath);
},
blockBlobPath,
new RetriableOperationExceptionHandler(AzureStorageExceptionHandler),
RetryCount));
}
catch (Exception e)
{
TraceSource.WriteExceptionAsError(
TraceType,
e,
"Failed to get block blob reference {0} from Azure storage container.",
blockBlobPath);
throw;
}
}
public static CloudBlobDirectory GetCloudBlobDirectoryRef(CloudBlobContainer container, string directoryPath)
{
try
{
return(BackupRestoreUtility.PerformWithRetries <string, CloudBlobDirectory>(
(string relDirName) =>
{
return container.GetDirectoryReference(relDirName);
},
directoryPath,
new RetriableOperationExceptionHandler(AzureStorageExceptionHandler),
RetryCount));
}
catch (Exception e)
{
TraceSource.WriteExceptionAsError(
TraceType,
e,
"Failed to get directory reference {0} from Azure storage.",
directoryPath);
throw;
}
}
/// <summary>
/// Gets called when a policy is updated
/// </summary>
internal async Task UpdatePolicyAsync(string backupPolicyName, TimeSpan timeout, CancellationToken cancellationToken, ITransaction transaction)
{
/**
* Cases:
* 1. UpdateRetention ----> Re-arm the timer.
* 0. Add Retention.
* a. Remove retention --> dispose the timer.
* b. update the retention.
* 2. UpdateStorage ---->
* 3. Update Backup Schedule. --> no need to do anything/
*
*/
try
{
BackupRestoreTrace.TraceSource.WriteInfo(TraceType, " In UpdatePolicyAsync of Retention Manager for Backup policy: {0}", backupPolicyName);
var backupPolicyStore = await BackupPolicyStore.CreateOrGetBackupPolicyStore(this.StatefulService);
RetentionMetadata retentionMetadata = await this.RetentionStore.GetValueAsync(backupPolicyName, timeout, cancellationToken, transaction);
BackupPolicy backupPolicy = await backupPolicyStore.GetValueAsync(backupPolicyName, timeout, cancellationToken, transaction);
if (retentionMetadata == null)
{
if (backupPolicy.RetentionPolicy != null)
{
retentionMetadata = new RetentionMetadata(backupPolicy.RetentionPolicy);
await this.RetentionStore.AddAsync(backupPolicyName, retentionMetadata, timeout, cancellationToken, transaction);
RetentionScheduler retentionScheduler = new RetentionScheduler(backupPolicy.Name, backupPolicy.RetentionPolicy, timerCallback);
// Here we have updated the policy to introduce retention.
retentionScheduler.ArmTimer(retentionMetadata, true);
if (!RetentionSchedulerDict.TryAdd(backupPolicyName, retentionScheduler))
{
// If we are not able to add retention in RetentionSchedulerDict then, stop timer in retention scheduler
retentionScheduler.Stop();
BackupRestoreTrace.TraceSource.WriteWarning(TraceType, "UpdatePolicyAsync, Not able to add retentionScheduler to dict for policy : {0} ", backupPolicyName);
throw new InvalidOperationException(string.Format("{0}: Key already exists.", backupPolicyName));
}
}
}
else if (backupPolicy.RetentionPolicy == null)
{
// need to remove the retentionScheduler from RetentionStore.
Func <bool> condition = () =>
{
if (retentionMetadata.LastRetentionCompletionTime > retentionMetadata.LastRetentionStartTime || retentionMetadata.LastRetentionStartTime == DateTime.MinValue)
{
return(true);
}
return(false);
};
bool disposeComplete = await BackupRestoreUtility.TryPerformOperationWithWaitAsync(
(policyName, token) =>
{
return(DisposeRetentionScheduler(policyName, timeout, token, transaction));
},
condition,
backupPolicyName,
cancellationToken,
minimumWaitTimeForRetryOperationInMs,
maxRetryCountForDisposingRetention,
minimumWaitTimeForRetryOperationInMs);
if (!disposeComplete)
{
BackupRestoreTrace.TraceSource.WriteError(TraceType, "Dispose retention did not complete for {0}", backupPolicyName);
throw new FabricBackupRestoreLocalRetryException();
}
}
else
{
if (retentionMetadata.UpdateRetentionPolicyMetadata(backupPolicy.RetentionPolicy))
{
// Update the retention Scheduler store and arm the timer.
RetentionScheduler retentionScheduler;
if (!RetentionSchedulerDict.TryGetValue(backupPolicyName, out retentionScheduler))
{
BackupRestoreTrace.TraceSource.WriteError(TraceType, "UpdateBackupPolicy: Not able to get retention scheduler for backupPolicy {0}", backupPolicyName);
throw new KeyNotFoundException();
}
retentionScheduler.RearmTimer(false, retentionMetadata);
await this.RetentionStore.UpdateValueAsync(backupPolicyName, retentionMetadata, timeout, cancellationToken, transaction);
}
}
}
catch (Exception ex)
{
if (ex is OperationCanceledException)
{
BackupRestoreTrace.TraceSource.WriteWarning(TraceType, " DisposeRetentionScheduler ended with Timeout as the operation was cencelled: {0}", backupPolicyName);
BackupRestoreTrace.TraceSource.WriteWarning(TraceType, " It could be because the retention was in progress and user tried to remove the retention policy during UpdatePolicy: {0}", backupPolicyName);
}
else
{
// Check for the exceptions of DisposeRetentionScheduler Timeout Retries.
BackupRestoreTrace.TraceSource.WriteError(TraceType, " UpdateBackupPolicy resulted in the exception : {0}", backupPolicyName);
}
throw;
}
}
