protected internal async Task <List <BackupRestoreView.RestorePoint> > GetBackupEnumerationsTask(string applicationName, string serviceName, string partitionId,
DateTime startDateTime, DateTime endDateTime, bool isLatestRequested, CancellationToken cancellationToken,
BRSContinuationToken continuationToken = null, int maxResults = 0)
{
var storeManager = RecoveryPointManagerFactory.GetRecoveryPointManager(this);
if (continuationToken == null)
{
continuationToken = new BRSContinuationToken();
}
var files = storeManager.EnumerateRecoveryPointMetadataFiles(GetRelativeStorePath(applicationName, serviceName, partitionId), startDateTime,
endDateTime, isLatestRequested, continuationToken, maxResults);
if ((files != null) && (files.Count > 0))
{
return(await storeManager.GetRecoveryPointDetailsAsync(files, cancellationToken));
}
return(null);
}
private async Task DeleteFilesForPartition(string partitionId, DateTime endDateFilter, int minimumNumberOfBackups, BackupStorage storage, string backupPolicyName, bool isCleanupTask = false)
{
// Design:
/**
* Need out Two API's from Azure share and file share
* 1. List of all backups before a certain date for the partition
* 2. Start deleting the files one by one.(one file at a time with some random wait time.)
*
**/
BackupRestoreTrace.TraceSource.WriteInfo(TraceType, " In DeleteFilesForPartition for partitionId : {0} with endDateFilter {1}", partitionId, endDateFilter);
string applicationName = null;
string serviceName = null;
string partitionID = null;
string fabricUri = await UtilityHelper.GetFabricUriFromPartitionId(partitionId, this.timeout, this.tokenSource.Token);
var FabricBackupResourceType = UtilityHelper.GetApplicationAndServicePartitionName(fabricUri, out applicationName,
out serviceName, out partitionID);
Debug.Assert(partitionId == partitionID);
var storeManager = RecoveryPointManagerFactory.GetRecoveryPointManager(storage);
this.tokenSource.Token.ThrowIfCancellationRequested();
BRSContinuationToken bRSContinuationToken = new BRSContinuationToken();
List <string> allBackupFiles = storeManager.EnumerateRecoveryPointMetadataFiles(storage.GetRelativeStorePath(applicationName, serviceName, partitionId),
DateTime.MinValue, DateTime.MaxValue, false, bRSContinuationToken, 0);
List <RestorePoint> restorePoints = new List <RestorePoint>();
if ((allBackupFiles != null) && (allBackupFiles.Count > 0))
{
restorePoints = await storeManager.GetRecoveryPointDetailsAsync(allBackupFiles, this.tokenSource.Token);
}
this.tokenSource.Token.ThrowIfCancellationRequested();
int backupCount = allBackupFiles.Count;
if (backupCount <= minimumNumberOfBackups)
{
return;
}
restorePoints = restorePoints.OrderBy(restorePoint => restorePoint.CreationTimeUtc).ToList();
restorePoints = restorePoints.Where(restorePoint => restorePoint.CreationTimeUtc < endDateFilter).ToList();
restorePoints.Reverse();
bool lastFullBackupFound = false;
RetentionMetadata retentionMetadata = await this.RetentionStore.GetValueAsync(backupPolicyName, this.timeout, this.tokenSource.Token);
if (isCleanupTask)
{
lastFullBackupFound = true;
}
if (!isCleanupTask && retentionMetadata.OnGoingRetentionFile.ContainsKey(partitionId) &&
retentionMetadata.OnGoingRetentionFile[partitionId] != null)
{
storeManager.DeleteBackupFiles(retentionMetadata.OnGoingRetentionFile[partitionId]);
}
foreach (var restorePoint in restorePoints)
{
this.tokenSource.Token.ThrowIfCancellationRequested();
BackupRestoreTrace.TraceSource.WriteInfo(TraceType, " Trying to Delete Restore point with backupLocation: {0}", restorePoint.BackupLocation);
if (!isCleanupTask)
{
BackupRestoreTrace.TraceSource.WriteInfo(TraceType, " Updating OnGoing RetentionFile in the metadata for backupPolicy : {0} with partitionId : {1} and BackupLocation {2}"
, backupPolicyName, partitionId, restorePoint.BackupLocation);
retentionMetadata.UpdateOnGoingRetentionFile(partitionId, restorePoint.BackupLocation);
await this.RetentionStore.UpdateValueAsync(backupPolicyName, retentionMetadata, this.timeout, this.tokenSource.Token, null);
}
if (!lastFullBackupFound)
{
BackupRestoreTrace.TraceSource.WriteInfo(TraceType, " Skipping RestorePoint with location : {0} as full backup is not found", restorePoint.BackupLocation);
if (restorePoint.BackupType == BackupOptionType.Full)
{
BackupRestoreTrace.TraceSource.WriteInfo(TraceType, " Full Backup found so, after restore points after :{0} will be deleted. ", restorePoint.BackupLocation);
lastFullBackupFound = true;
}
}
else
{
if (backupCount <= minimumNumberOfBackups)
{
break;
}
if (storeManager.DeleteBackupFiles(restorePoint.BackupLocation))
{
BackupRestoreTrace.TraceSource.WriteInfo(TraceType, " Deleted Restore point with backupLocation: {0}", restorePoint.BackupLocation);
backupCount--;
}
else
{
BackupRestoreTrace.TraceSource.WriteWarning(TraceType, " Not able to delete Restore point with backupLocation: {0}", restorePoint.BackupLocation);
}
}
await IntroduceRandomDelay();
}
// Deletion for a partition is completed.
if (!isCleanupTask)
{
BackupRestoreTrace.TraceSource.WriteInfo(TraceType, " Updating OnGoing RetentionFile in the metadata for backupPolicy : {0}", backupPolicyName);
retentionMetadata.UpdateOnGoingRetentionFile(partitionId);
await this.RetentionStore.UpdateValueAsync(backupPolicyName, retentionMetadata, this.timeout, this.tokenSource.Token, null);
}
}
internal override async Task <HttpResponseMessage> RunAsync(TimeSpan timeout, CancellationToken cancellationToken)
{
string applicationNameUri;
string serviceNameUri;
string partitionId;
var fabricUri = await UtilityHelper.GetFabricUriFromRequstHeaderForPartitions(this.fabricRequestHeader, timeout, cancellationToken);
this.ThrowInvalidArgumentIfNull(this.restoreRequest);
await FabricClientHelper.IsFaultServiceExisting(timeout, cancellationToken);
UtilityHelper.GetApplicationAndServicePartitionUri(fabricUri, out applicationNameUri, out serviceNameUri, out partitionId);
BackupRestoreModel.BackupStorage backupStorage;
if (this.restoreRequest.BackupStorage == null)
{
var backupProtection =
await this.BackupMappingStore.GetValueAsync(fabricUri, timeout, cancellationToken) ??
await this.BackupMappingStore.GetValueAsync(serviceNameUri, timeout, cancellationToken) ??
await this.BackupMappingStore.GetValueAsync(applicationNameUri, timeout, cancellationToken);
if (backupProtection == null)
{
throw new FabricPeriodicBackupNotEnabledException();
}
var backupPolicy =
await this.BackupPolicyStore.GetValueAsync(backupProtection.BackupPolicyName, timeout, cancellationToken);
backupStorage = backupPolicy.Storage;
}
else
{
backupStorage = BackupRestoreModel.BackupStorage.FromBackupStorageView(this.restoreRequest.BackupStorage);
}
var recoveryPointManager = RecoveryPointManagerFactory.GetRecoveryPointManager(backupStorage);
var backupDetails = await recoveryPointManager.GetRecoveryPointDetailsAsync(this.restoreRequest.BackupLocation, cancellationToken);
var retriveBackupDetails = backupDetails as RestorePoint;
if (retriveBackupDetails == null)
{
throw new FabricException(StringResources.RestoreDetailsFetchFailed, FabricErrorCode.RestoreSafeCheckFailed);
}
if (!retriveBackupDetails.BackupId.Equals(this.restoreRequest.BackupId))
{
/*
* Add validation here for the request
*/
throw new FabricException(StringResources.BackupIdAndLocationMismatch,
FabricErrorCode.RestoreSourceTargetPartitionMismatch);
}
var backupServicePartitionInformation = retriveBackupDetails.PartitionInformation;
var servicePartitionInformation = await FabricClientHelper.GetPartitionDetails(partitionId, timeout, cancellationToken);
switch (servicePartitionInformation.Kind)
{
case ServicePartitionKind.Int64Range:
var int64RangePartitionInformation = servicePartitionInformation as System.Fabric.Int64RangePartitionInformation;
var backupInt64RangePartitionInformation = backupServicePartitionInformation as BackupRestoreTypes.Int64RangePartitionInformation;
if (backupInt64RangePartitionInformation == null ||
backupInt64RangePartitionInformation.HighKey != int64RangePartitionInformation.HighKey ||
backupInt64RangePartitionInformation.LowKey != int64RangePartitionInformation.LowKey)
{
throw new FabricException(StringResources.InvalidInt64Keys,
FabricErrorCode.RestoreSourceTargetPartitionMismatch);
}
break;
case ServicePartitionKind.Named:
var namedPartitionInformation = servicePartitionInformation as System.Fabric.NamedPartitionInformation;
var backupNamedPartitionInformation = backupServicePartitionInformation as BackupRestoreTypes.NamedPartitionInformation;
if (backupNamedPartitionInformation == null ||
!namedPartitionInformation.Name.Equals(backupNamedPartitionInformation.Name))
{
throw new FabricException(StringResources.InvalidNameKey,
FabricErrorCode.RestoreSourceTargetPartitionMismatch);
}
break;
case ServicePartitionKind.Singleton:
var backupSingletonPartitionInformation = backupServicePartitionInformation as BackupRestoreTypes.SingletonPartitionInformation;
if (backupSingletonPartitionInformation == null)
{
throw new FabricException(StringResources.InvalidPartitionType,
FabricErrorCode.RestoreSourceTargetPartitionMismatch);
}
break;
}
var backupLocations =
await recoveryPointManager.GetBackupLocationsInBackupChainAsync(this.restoreRequest.BackupLocation, cancellationToken);
var dataLossGuid = Guid.NewGuid();
var restoreRequestGuid = Guid.NewGuid();
var restoreStatus = new RestoreStatus(fabricUri, backupStorage, restoreRequestGuid, dataLossGuid, backupLocations, retriveBackupDetails);
var restoreWorkItem = new RestorePartitionWorkItem(serviceNameUri, partitionId, restoreRequestGuid, dataLossGuid, this.restoreTimeoutInMinutes);
using (var transaction = this.StatefulService.StateManager.CreateTransaction())
{
await this.CheckForEitherBackupOrRestoreInProgress(fabricUri, timeout, cancellationToken, transaction);
await this.WorkItemQueue.AddWorkItem(restoreWorkItem, timeout, cancellationToken, transaction);
await this.RestoreStore.AddOrUpdateAsync(fabricUri, restoreStatus,
(fabricUri1, restoreStatus1) => restoreStatus, timeout, cancellationToken, transaction);
await transaction.CommitAsync();
}
return(new HttpResponseMessage(HttpStatusCode.Accepted));
}
public async Task <RestorePointDetails> GetRestorePointDetailsAsync(BackupPartitionInfo backupPartitionInfo, TimeSpan timeout,
CancellationToken cancellationToken)
{
BackupRestoreTrace.TraceSource.WriteInfo(TraceType, "GetRestorePointDetailsAsync invoked for partition {0}", backupPartitionInfo.PartitionId);
var restoreStore = await RestoreStore.CreateOrGetRestoreStatusStore(this.statefulService);
var backupMappingStore = await BackupMappingStore.CreateOrGetBackupMappingStore(this.statefulService);
var backupPolicyStore = await BackupPolicyStore.CreateOrGetBackupPolicyStore(this.statefulService);
bool userInitiatedOperation;
BackupStorage backupStorage;
BackupStoreInformation backupStoreInformation = null;
List <string> backupLocations;
string brsServiceUri = await UtilityHelper.GetCustomServiceUri(backupPartitionInfo.ServiceName, timeout, cancellationToken);
var fabricUri = UtilityHelper.GetBackupMappingKey(brsServiceUri, backupPartitionInfo.PartitionId.ToString());
var restoreStatus = await restoreStore.GetValueAsync(fabricUri);
if (restoreStatus != null && (
restoreStatus.RestoreStatusState != RestoreState.Failure ||
restoreStatus.RestoreStatusState != RestoreState.Invalid ||
restoreStatus.RestoreStatusState != RestoreState.Success ||
restoreStatus.RestoreStatusState != RestoreState.Timeout))
{
backupStorage = restoreStatus.BackupStorage;
userInitiatedOperation = true;
backupLocations = restoreStatus.BackupLocations.ToList();
}
else
{
var backupMappingKey = UtilityHelper.GetBackupMappingKey(brsServiceUri,
backupPartitionInfo.PartitionId.ToString());
var backupMappingModel = (await backupMappingStore.GetValueAsync(backupMappingKey) ??
await backupMappingStore.GetValueAsync(brsServiceUri)) ??
await backupMappingStore.GetValueAsync(UtilityHelper.GetApplicationNameFromService(brsServiceUri));
if (backupMappingModel == null)
{
throw new FabricPeriodicBackupNotEnabledException();
}
var backupPolicy = await backupPolicyStore.GetValueAsync(backupMappingModel.BackupPolicyName);
if (backupPolicy != null && backupPolicy.AutoRestore)
{
backupStorage = backupPolicy.Storage;
userInitiatedOperation = false;
string applicationName, serviceName;
UtilityHelper.GetApplicationAndServiceNameFromServiceUri(brsServiceUri,
out applicationName, out serviceName);
// Get the latest backup now
var backups = await backupStorage.GetBackupEnumerationsTask(
applicationName,
serviceName,
backupPartitionInfo.PartitionId.ToString(),
DateTime.MinValue,
DateTime.MaxValue,
true,
cancellationToken);
if (backups == null || backups.Count == 0)
{
throw new ArgumentException(); // TODO: Change this with appropriate exception
}
var backupDetail = backups.First();
var recoveryPointManager = RecoveryPointManagerFactory.GetRecoveryPointManager(backupStorage);
backupLocations = await recoveryPointManager.GetBackupLocationsInBackupChainAsync(backupDetail.BackupLocation, cancellationToken);
}
else
{
throw new ArgumentException(); // TODO: Change with the OTher exception
}
}
switch (backupStorage.BackupStorageType)
{
case BackupStorageType.FileShare:
var fileShareStorage = (FileShareStorage)backupStorage;
backupStoreInformation = new FileShareBackupStore
{
AccessType = String.IsNullOrEmpty(fileShareStorage.PrimaryUserName) ? FileShareAccessType.None : FileShareAccessType.DomainUser,
FileSharePath = fileShareStorage.Path,
PrimaryUserName = fileShareStorage.PrimaryUserName,
PrimaryPassword = fileShareStorage.PrimaryPassword,
SecondaryUserName = fileShareStorage.SecondaryUserName,
SecondaryPassword = fileShareStorage.SecondaryPassword,
IsPasswordEncrypted = fileShareStorage.IsPasswordEncrypted,
};
break;
case BackupStorageType.AzureBlobStore:
var azureStorage = (AzureStorage)backupStorage;
backupStoreInformation = new AzureBlobBackupStore
{
ConnectionString = azureStorage.ConnectionString,
ContainerName = azureStorage.ContainerName,
FolderPath = String.Empty, // TODO: This should be constructed
IsAccountKeyEncrypted = azureStorage.IsConnectionStringEncrypted,
};
break;
case BackupStorageType.DsmsAzureBlobStore:
var dsmsAzureStorage = (DsmsAzureStorage)backupStorage;
backupStoreInformation = new DsmsAzureBlobBackupStore
{
StorageCredentialsSourceLocation = dsmsAzureStorage.StorageCredentialsSourceLocation,
ContainerName = dsmsAzureStorage.ContainerName,
FolderPath = String.Empty, // TODO: This should be constructed
};
break;
}
Guid restoreOperationId = Guid.NewGuid();
if (userInitiatedOperation)
{
await UtilityHelper.InvokeWithRetry(async() =>
{
using (ITransaction transaction = this.statefulService.StateManager.CreateTransaction())
{
var currentRestoreStatus = await restoreStore.GetValueWithUpdateLockModeAsync(fabricUri, timeout, cancellationToken, transaction);
restoreOperationId = currentRestoreStatus.RestoreRequestGuid;
if (currentRestoreStatus.RestoreStatusState == RestoreState.Accepted) //TODO Check for Guid
{
var toUpdateRestoreStatus = currentRestoreStatus.ToBuilder().WithState(RestoreState.RestoreInProgress).Build();
await
restoreStore.UpdateValueAsync(fabricUri, toUpdateRestoreStatus, timeout,
cancellationToken, transaction);
await transaction.CommitAsync();
}
else
{
transaction.Abort();
}
}
});
}
var restorePartitionDetails = new RestorePointDetails()
{
BackupLocations = backupLocations,
UserInitiatedOperation = userInitiatedOperation,
StoreInformation = backupStoreInformation,
OperationId = restoreOperationId
};
BackupRestoreTrace.TraceSource.WriteInfo(TraceType, "GetRestorePointDetailsAsync invoked for partition {0} with Return Values {1}", backupPartitionInfo.PartitionId, restorePartitionDetails);
return(restorePartitionDetails);
}
