// This encapsulates a restore task which triggers the restore and returns the restore result accordingly.
public async Task <RestoreResult> RestoreWorkFlow(JToken latestbackupInfo, string policy, PartitionWrapper partition, String clusterConnectionString, String clusterThumbprint)
{
string URL = clusterConnectionString + "/";
string URLParameters = "Partitions/" + partition.partitionId + "/$/Restore" + "?api-version=6.4";
BackupStorage backupStorage = await GetBackupStorageDetails(policy);
if (backupStorage == null)
{
ServiceEventSource.Current.ServiceMessage(this.Context, "backupstorage is null");
return(null);
}
BackupInfo backupInfo = new BackupInfo(latestbackupInfo["BackupId"].ToString(), latestbackupInfo["BackupLocation"].ToString(), backupStorage, (DateTime)latestbackupInfo["CreationTimeUtc"]);
HttpResponseMessage response = await Utility.HTTPPostAsync(URL, URLParameters, backupInfo, clusterThumbprint);
if (response.IsSuccessStatusCode || response.StatusCode == System.Net.HttpStatusCode.Conflict) // As calling Restore multiple times results in 409/Conflict Error, when in progress
{
string restoreState = "";
restoreState = GetRestoreState(partition, clusterConnectionString, clusterThumbprint);
return(new RestoreResult(backupInfo, restoreState));
}
else
{
Console.WriteLine("{0} ({1})", (int)response.StatusCode, response.ReasonPhrase);
return(null);
}
}
// This encapsulates a restore task which triggers the restore and returns the restore result accordingly.
public async Task <RestoreResult> RestoreWorkFlow(JToken latestbackupInfo, string policy, PartitionWrapper partition, String clusterConnectionString, String clusterThumbprint, String partitionDictionary)
{
IReliableDictionary <Guid, PartitionWrapper> myDictionary = await this.StateManager.GetOrAddAsync <IReliableDictionary <Guid, PartitionWrapper> >(partitionDictionary);
string URL = clusterConnectionString + "/";
string urlParameters = "Partitions/" + partition.partitionId + "/$/Restore" + "?api-version=6.2-preview";
X509Certificate2 clientCert = GetClientCertificate(clusterThumbprint);
WebRequestHandler requestHandler = new WebRequestHandler();
requestHandler.ClientCertificates.Add(clientCert);
requestHandler.ServerCertificateValidationCallback = this.MyRemoteCertificateValidationCallback;
HttpClient client = new HttpClient(requestHandler)
{
BaseAddress = new Uri(URL)
};
client.DefaultRequestHeaders.Accept.Add(
new MediaTypeWithQualityHeaderValue("application/json"));
BackupStorage backupStorage = await GetBackupStorageDetails(policy);
if (backupStorage == null)
{
ServiceEventSource.Current.ServiceMessage(this.Context, "backupstorage is null");
return(null);
}
BackupInfo backupInfo = new BackupInfo(latestbackupInfo["BackupId"].ToString(), latestbackupInfo["BackupLocation"].ToString(), backupStorage, (DateTime)latestbackupInfo["CreationTimeUtc"]);
HttpResponseMessage response = await client.PostAsJsonAsync(urlParameters, backupInfo); // Blocking call!
if (response.IsSuccessStatusCode || response.StatusCode == System.Net.HttpStatusCode.Conflict) // As calling Restore multiple times results in 409/Conflict Error, when in progress
{
string restoreState = "";
//do
//{
// await Task.Delay(10000);
// restoreState = GetRestoreState(partition, clusterConnectionString, clusterThumbprint);
//} while (restoreState.Equals("Accepted") || restoreState.Equals("RestoreInProgress"));
restoreState = GetRestoreState(partition, clusterConnectionString, clusterThumbprint);
return(new RestoreResult(backupInfo, restoreState));
}
else
{
Console.WriteLine("{0} ({1})", (int)response.StatusCode, response.ReasonPhrase);
return(null);
}
}
// This encapsulates a restore task which triggers the restore and returns the restore result accordingly.
public async Task <RestoreResult> RestoreWorkFlow(JToken latestbackupInfo, string policy, PartitionWrapper partition, String clusterConnectionString, String partitionDictionary)
{
IReliableDictionary <Guid, PartitionWrapper> myDictionary = await this.StateManager.GetOrAddAsync <IReliableDictionary <Guid, PartitionWrapper> >(partitionDictionary);
HttpClient client = new HttpClient();
string URL = clusterConnectionString + "/Partitions/" + partition.partitionId + "/$/Restore";
string urlParameters = "?api-version=6.2-preview";
client.BaseAddress = new Uri(URL);
client.DefaultRequestHeaders.Accept.Add(
new MediaTypeWithQualityHeaderValue("application/json"));
BackupStorage backupStorage = await GetBackupStorageDetails(policy);
if (backupStorage == null)
{
ServiceEventSource.Current.ServiceMessage(this.Context, "backupstorage is null");
return(null);
}
BackupInfo backupInfo = new BackupInfo(latestbackupInfo["BackupId"].ToString(), latestbackupInfo["BackupLocation"].ToString(), backupStorage, (DateTime)latestbackupInfo["CreationTimeUtc"]);
HttpResponseMessage response = await client.PostAsJsonAsync(urlParameters, backupInfo); // Blocking call!
if (response.IsSuccessStatusCode)
{
string restoreState = "";
do
{
await Task.Delay(10000);
restoreState = GetRestoreState(partition, clusterConnectionString);
} while (restoreState.Equals("Accepted") || restoreState.Equals("RestoreInProgress"));
return(new RestoreResult(backupInfo, restoreState));
}
else
{
Console.WriteLine("{0} ({1})", (int)response.StatusCode, response.ReasonPhrase);
return(null);
}
}
public async Task OnTimerTick()
{
// On every timer tick calls this method which goes through the workflowsInProgress dictionary
// and removes the completed tasks and updates the partition metadata.
// For every partition mapping in the reliable dictionary if the corresponding task is not present in the
// workflowsInProgress dictionary it will create a task and puts in the dictionary
IReliableDictionary <Guid, PartitionWrapper> myDictionary = await this.StateManager.GetOrAddAsync <IReliableDictionary <Guid, PartitionWrapper> >("partitionDictionary");
List <Guid> keysToRemove = new List <Guid>();
if (workFlowsInProgress.Count != 0)
{
try
{
foreach (KeyValuePair <Guid, Task <RestoreResult> > workFlow in workFlowsInProgress)
{
Task <RestoreResult> task = workFlow.Value;
if (task.IsCompleted)
{
RestoreResult restoreResult = task.Result;
using (ITransaction tx = this.StateManager.CreateTransaction())
{
ConditionalValue <PartitionWrapper> partitionWrapper = await myDictionary.TryGetValueAsync(tx, workFlow.Key);
if (partitionWrapper.HasValue)
{
if (restoreResult == null)
{
PartitionWrapper updatedPartitionWrapper = ObjectExtensions.Copy(partitionWrapper.Value);
updatedPartitionWrapper.CurrentlyUnderRestore = null;
await myDictionary.SetAsync(tx, workFlow.Key, updatedPartitionWrapper);
ServiceEventSource.Current.ServiceMessage(this.Context, "Restore Task returned null!!! ");
}
else if (restoreResult.restoreState.Equals("Success"))
{
PartitionWrapper updatedPartitionWrapper = ObjectExtensions.Copy(partitionWrapper.Value);
updatedPartitionWrapper.LastBackupRestored = restoreResult.restoreInfo;
updatedPartitionWrapper.CurrentlyUnderRestore = null;
await myDictionary.SetAsync(tx, workFlow.Key, updatedPartitionWrapper);
ServiceEventSource.Current.ServiceMessage(this.Context, "Restored succcessfully!!! ");
}
}
await tx.CommitAsync();
}
keysToRemove.Add(workFlow.Key);
}
}
foreach (var key in keysToRemove)
{
workFlowsInProgress.Remove(key);
}
}
catch (Exception ex)
{
ServiceEventSource.Current.Message("exception caught : {0}", ex);
}
}
using (ITransaction tx = this.StateManager.CreateTransaction())
{
IAsyncEnumerable <KeyValuePair <Guid, PartitionWrapper> > enumerable = await myDictionary.CreateEnumerableAsync(tx);
IAsyncEnumerator <KeyValuePair <Guid, PartitionWrapper> > asyncEnumerator = enumerable.GetAsyncEnumerator();
while (await asyncEnumerator.MoveNextAsync(CancellationToken.None))
{
Guid primaryPartition = asyncEnumerator.Current.Key;
PartitionWrapper secondaryPartition = asyncEnumerator.Current.Value;
if (secondaryPartition == null)
{
continue;
}
JToken backupInfoToken = await GetLatestBackupAvailable(primaryPartition, "http://" + secondaryPartition.primaryCluster.address + ":" + secondaryPartition.primaryCluster.httpEndpoint);
if (backupInfoToken == null)
{
continue;
}
BackupInfo backupInfo = new BackupInfo(backupInfoToken["BackupId"].ToString(), backupInfoToken["BackupLocation"].ToString(), (DateTime)backupInfoToken["CreationTimeUtc"]);
string backupPolicy = await GetPolicy("http://" + secondaryPartition.primaryCluster.address + ":" + secondaryPartition.primaryCluster.httpEndpoint, primaryPartition);
if (backupPolicy == null)
{
continue;
}
Task <RestoreResult> task = workFlowsInProgress.TryGetValue(primaryPartition, out Task <RestoreResult> value) ? value : null;
if (task == null)
{
if (secondaryPartition.LastBackupRestored == null || DateTime.Compare(backupInfo.backupTime, secondaryPartition.LastBackupRestored.backupTime) > 0)
{
Task <RestoreResult> restoreTask = Task <RestoreResult> .Run(() => RestoreWorkFlow(backupInfoToken, backupPolicy, secondaryPartition, "http://" + secondaryPartition.secondaryCluster.address + ":" + secondaryPartition.secondaryCluster.httpEndpoint, "partitionDictionary"));
workFlowsInProgress.Add(asyncEnumerator.Current.Key, restoreTask);
PartitionWrapper updatedPartitionWrapper = ObjectExtensions.Copy(secondaryPartition);
updatedPartitionWrapper.LatestBackupAvailable = backupInfo;
updatedPartitionWrapper.CurrentlyUnderRestore = backupInfo;
await myDictionary.SetAsync(tx, primaryPartition, updatedPartitionWrapper);
}
else
{
continue;
}
}
else if (task.IsCompleted)
{
RestoreResult restoreResult = task.Result;
if (restoreResult.restoreState.Equals("Success"))
{
PartitionWrapper updatedPartitionWrapper = ObjectExtensions.Copy(secondaryPartition);
updatedPartitionWrapper.LastBackupRestored = restoreResult.restoreInfo;
updatedPartitionWrapper.CurrentlyUnderRestore = null;
await myDictionary.SetAsync(tx, primaryPartition, updatedPartitionWrapper);
ServiceEventSource.Current.ServiceMessage(this.Context, "Successfully Restored!!! ");
}
workFlowsInProgress.Remove(primaryPartition);
if (secondaryPartition.LastBackupRestored == null || DateTime.Compare(backupInfo.backupTime, secondaryPartition.LastBackupRestored.backupTime) > 0)
{
Task <RestoreResult> restoreTask = Task <string> .Run(() => RestoreWorkFlow(backupInfoToken, backupPolicy, secondaryPartition, "http://" + secondaryPartition.secondaryCluster.address + ":" + secondaryPartition.secondaryCluster.httpEndpoint, "partitionDictionary"));
workFlowsInProgress.Add(primaryPartition, restoreTask);
PartitionWrapper updatedPartitionWrapper = ObjectExtensions.Copy(secondaryPartition);
updatedPartitionWrapper.LatestBackupAvailable = backupInfo;
updatedPartitionWrapper.CurrentlyUnderRestore = backupInfo;
await myDictionary.SetAsync(tx, primaryPartition, updatedPartitionWrapper);
}
else
{
continue;
}
}
else
{
PartitionWrapper updatedPartitionWrapper = ObjectExtensions.Copy(secondaryPartition);
updatedPartitionWrapper.LatestBackupAvailable = backupInfo;
await myDictionary.SetAsync(tx, primaryPartition, updatedPartitionWrapper);
}
}
await tx.CommitAsync();
}
}
public RestoreResult(BackupInfo restoreInfo, string restoreState)
{
this.restoreInfo = restoreInfo;
this.restoreState = restoreState;
}
