/// <summary>
/// This method fetches the restore state of each restore flow task
/// </summary>
/// <param name="partition">Which is under restore</param>
/// <param name="clusterConnectionString">Secondary Cluster String</param>
/// <returns></returns>
public string GetRestoreState(PartitionWrapper partition, string clusterConnectionString)
{
string URL = clusterConnectionString + "/Partitions/" + partition.partitionId + "/$/GetRestoreProgress";
string urlParameters = "?api-version=6.2-preview";
HttpClient client = new HttpClient();
client.BaseAddress = new Uri(URL);
client.DefaultRequestHeaders.Accept.Add(
new MediaTypeWithQualityHeaderValue("application/json"));
HttpResponseMessage response = client.GetAsync(urlParameters).Result;
if (response.IsSuccessStatusCode)
{
var content = response.Content.ReadAsAsync <JObject>().Result;
string restoreState = content["RestoreState"].ToString();
return(restoreState);
}
else
{
Console.WriteLine("{0} ({1})", (int)response.StatusCode, response.ReasonPhrase);
return(null);
}
}
// An interface method which returns the entries of reliable dictionary
public async Task <List <PartitionWrapper> > GetStatus()
{
IReliableDictionary <Guid, PartitionWrapper> myDictionary = await this.StateManager.GetOrAddAsync <IReliableDictionary <Guid, PartitionWrapper> >("partitionDictionary");
List <PartitionWrapper> mappedPartitions = new List <PartitionWrapper>();
using (ITransaction tx = this.StateManager.CreateTransaction())
{
IAsyncEnumerable <KeyValuePair <Guid, PartitionWrapper> > enumerable = await myDictionary.CreateEnumerableAsync(tx);
IAsyncEnumerator <KeyValuePair <Guid, PartitionWrapper> > asyncEnumerator = enumerable.GetAsyncEnumerator();
while (await asyncEnumerator.MoveNextAsync(CancellationToken.None))
{
ConditionalValue <PartitionWrapper> partitionWrapper = await myDictionary.TryGetValueAsync(tx, asyncEnumerator.Current.Key);
if (partitionWrapper.HasValue)
{
PartitionWrapper mappedPartition = partitionWrapper.Value;
mappedPartitions.Add(mappedPartition);
ServiceEventSource.Current.ServiceMessage(this.Context, "Successfully Retrieved!!! ");
}
}
await tx.CommitAsync();
}
return(mappedPartitions);
}
public async Task <List <Tuple <ClusterDetails, ClusterDetails> > > GetClusterCombinations()
{
HashSet <String> primarySecondarySet = new HashSet <String>();
List <Tuple <ClusterDetails, ClusterDetails> > clusterCombinations = new List <Tuple <ClusterDetails, ClusterDetails> >();
IReliableDictionary <String, PartitionWrapper> partitionDictionary = await this.StateManager.GetOrAddAsync <IReliableDictionary <String, PartitionWrapper> >("partitionDictionary");
using (ITransaction tx = this.StateManager.CreateTransaction())
{
IAsyncEnumerable <KeyValuePair <String, PartitionWrapper> > enumerable = await partitionDictionary.CreateEnumerableAsync(tx);
IAsyncEnumerator <KeyValuePair <String, PartitionWrapper> > asyncEnumerator = enumerable.GetAsyncEnumerator();
while (await asyncEnumerator.MoveNextAsync(CancellationToken.None))
{
String primarySecondaryJoinKey = asyncEnumerator.Current.Key;
PartitionWrapper partitionInfo = asyncEnumerator.Current.Value;
String primarySecondary = Utility.getPrimarySecondary(primarySecondaryJoinKey);
if (!primarySecondarySet.Contains(primarySecondary))
{
Tuple <ClusterDetails, ClusterDetails> clusterTuple = Tuple.Create(partitionInfo.primaryCluster, partitionInfo.secondaryCluster);
clusterCombinations.Add(clusterTuple);
primarySecondarySet.Add(primarySecondary);
}
}
await tx.CommitAsync();
}
return(clusterCombinations);
}
public PartitionWrapper(PartitionWrapper partitionWrapper)
{
this.partitionId = partitionWrapper.PartitionInformation.Id;
this.PartitionInformation = partitionWrapper.PartitionInformation;
this.ServiceKind = partitionWrapper.ServiceKind;
this.HealthState = partitionWrapper.HealthState;
this.PartitionStatus = partitionWrapper.PartitionStatus;
}
public async Task <string> DisconfigureService(string serviceName, string primaryCluster, string secondaryCluster)
{
List <String> keysToRemove = new List <String>();
IReliableDictionary <String, PartitionWrapper> myDictionary = await this.StateManager.GetOrAddAsync <IReliableDictionary <String, PartitionWrapper> >("partitionDictionary");
using (ITransaction tx = this.StateManager.CreateTransaction())
{
IAsyncEnumerable <KeyValuePair <String, PartitionWrapper> > enumerable = await myDictionary.CreateEnumerableAsync(tx);
IAsyncEnumerator <KeyValuePair <String, PartitionWrapper> > asyncEnumerator = enumerable.GetAsyncEnumerator();
while (await asyncEnumerator.MoveNextAsync(CancellationToken.None))
{
PartitionWrapper secondaryPartition = asyncEnumerator.Current.Value;
String partitionAccessKey = asyncEnumerator.Current.Key;
if (Utility.isPartitionFromPrimarySecondaryCombination(partitionAccessKey, primaryCluster, secondaryCluster))
{
if (secondaryPartition.serviceName.ToString().Equals(serviceName))
{
keysToRemove.Add(asyncEnumerator.Current.Key);
}
}
}
await tx.CommitAsync();
}
bool allPartitionsRemoved = true;
using (ITransaction tx = this.StateManager.CreateTransaction())
{
foreach (String key in keysToRemove)
{
ConditionalValue <PartitionWrapper> value = myDictionary.TryRemoveAsync(tx, key).Result;
if (!value.HasValue)
{
allPartitionsRemoved = false;
}
}
await tx.CommitAsync();
}
if (allPartitionsRemoved)
{
return(serviceName);
}
return(null);
}
/// <summary>
/// This method fetches the restore state of each restore flow task
/// </summary>
/// <param name="partition">Which is under restore</param>
/// <param name="clusterConnectionString">Secondary Cluster String</param>
/// <returns></returns>
public string GetRestoreState(PartitionWrapper partition, string clusterConnectionString, string clusterThumbprint)
{
string URL = clusterConnectionString + "/";
string URLParameters = "Partitions/" + partition.partitionId + "/$/GetRestoreProgress" + "?api-version=6.4";
HttpResponseMessage response = Utility.HTTPGetAsync(URL, URLParameters, clusterThumbprint).Result;
if (response.IsSuccessStatusCode)
{
var content = response.Content.ReadAsAsync <JObject>().Result;
string restoreState = (content != null) ? content["RestoreState"].ToString() : "";
return(restoreState);
}
else
{
Console.WriteLine("{0} ({1})", (int)response.StatusCode, response.ReasonPhrase);
return(null);
}
}
// An interface method which is for disconfiguring the appliations thereby deleting their entries in reliable dictionary.
public async Task <string> Disconfigure(string applicationName)
{
List <Guid> keysToRemove = new List <Guid>();
IReliableDictionary <Guid, PartitionWrapper> myDictionary = await this.StateManager.GetOrAddAsync <IReliableDictionary <Guid, PartitionWrapper> >("partitionDictionary");
using (ITransaction tx = this.StateManager.CreateTransaction())
{
IAsyncEnumerable <KeyValuePair <Guid, PartitionWrapper> > enumerable = await myDictionary.CreateEnumerableAsync(tx);
IAsyncEnumerator <KeyValuePair <Guid, PartitionWrapper> > asyncEnumerator = enumerable.GetAsyncEnumerator();
while (await asyncEnumerator.MoveNextAsync(CancellationToken.None))
{
PartitionWrapper secondaryPartition = asyncEnumerator.Current.Value;
if (secondaryPartition.applicationName.ToString().Equals(applicationName))
{
keysToRemove.Add(asyncEnumerator.Current.Key);
}
}
await tx.CommitAsync();
}
bool allPartitionsRemoved = true;
using (ITransaction tx = this.StateManager.CreateTransaction())
{
foreach (Guid key in keysToRemove)
{
ConditionalValue <PartitionWrapper> value = myDictionary.TryRemoveAsync(tx, key).Result;
if (!value.HasValue)
{
allPartitionsRemoved = false;
}
}
await tx.CommitAsync();
}
if (allPartitionsRemoved)
{
return(applicationName);
}
return(null);
}
/// <summary>
/// This method fetches the restore state of each restore flow task
/// </summary>
/// <param name="partition">Which is under restore</param>
/// <param name="clusterConnectionString">Secondary Cluster String</param>
/// <returns></returns>
public string GetRestoreState(PartitionWrapper partition, string clusterConnectionString, string clusterThumbprint)
{
string URL = clusterConnectionString + "/";
string urlParameters = "Partitions/" + partition.partitionId + "/$/GetRestoreProgress" + "?api-version=6.2-preview";
X509Certificate2 clientCert = GetClientCertificate(clusterThumbprint);
WebRequestHandler requestHandler = new WebRequestHandler();
requestHandler.ClientCertificates.Add(clientCert);
requestHandler.ServerCertificateValidationCallback = this.MyRemoteCertificateValidationCallback;
HttpClient client = new HttpClient(requestHandler)
{
BaseAddress = new Uri(URL)
};
client.DefaultRequestHeaders.Accept.Add(
new MediaTypeWithQualityHeaderValue("application/json"));
// List data response.
HttpResponseMessage response = client.GetAsync(urlParameters).Result; // Blocking call!
if (response.IsSuccessStatusCode)
{
var content = response.Content.ReadAsAsync <JObject>().Result;
string restoreState = content["RestoreState"].ToString();
return(restoreState);
}
else
{
Console.WriteLine("{0} ({1})", (int)response.StatusCode, response.ReasonPhrase);
return(null);
}
}
public async Task OnTimerTick()
{
// On every timer tick calls this method which goes through the workflowsInProgress dictionary
// and removes the completed tasks and updates the partition metadata.
// For every partition mapping in the reliable dictionary if the corresponding task is not present in the
// workflowsInProgress dictionary it will create a task and puts in the dictionary
IReliableDictionary <Guid, PartitionWrapper> myDictionary = await this.StateManager.GetOrAddAsync <IReliableDictionary <Guid, PartitionWrapper> >("partitionDictionary");
List <Guid> keysToRemove = new List <Guid>();
if (workFlowsInProgress.Count != 0)
{
try
{
foreach (KeyValuePair <Guid, Task <RestoreResult> > workFlow in workFlowsInProgress)
{
Task <RestoreResult> task = workFlow.Value;
if (task.IsCompleted)
{
RestoreResult restoreResult = task.Result;
using (ITransaction tx = this.StateManager.CreateTransaction())
{
ConditionalValue <PartitionWrapper> partitionWrapper = await myDictionary.TryGetValueAsync(tx, workFlow.Key);
if (partitionWrapper.HasValue)
{
if (restoreResult == null)
{
PartitionWrapper updatedPartitionWrapper = ObjectExtensions.Copy(partitionWrapper.Value);
updatedPartitionWrapper.CurrentlyUnderRestore = null;
await myDictionary.SetAsync(tx, workFlow.Key, updatedPartitionWrapper);
ServiceEventSource.Current.ServiceMessage(this.Context, "Restore Task returned null!!! ");
}
else if (restoreResult.restoreState.Equals("Success"))
{
PartitionWrapper updatedPartitionWrapper = ObjectExtensions.Copy(partitionWrapper.Value);
updatedPartitionWrapper.LastBackupRestored = restoreResult.restoreInfo;
updatedPartitionWrapper.CurrentlyUnderRestore = null;
await myDictionary.SetAsync(tx, workFlow.Key, updatedPartitionWrapper);
ServiceEventSource.Current.ServiceMessage(this.Context, "Restored succcessfully!!! ");
}
}
await tx.CommitAsync();
}
keysToRemove.Add(workFlow.Key);
}
}
foreach (var key in keysToRemove)
{
workFlowsInProgress.Remove(key);
}
}
catch (Exception ex)
{
ServiceEventSource.Current.Message("exception caught : {0}", ex);
}
}
using (ITransaction tx = this.StateManager.CreateTransaction())
{
IAsyncEnumerable <KeyValuePair <Guid, PartitionWrapper> > enumerable = await myDictionary.CreateEnumerableAsync(tx);
IAsyncEnumerator <KeyValuePair <Guid, PartitionWrapper> > asyncEnumerator = enumerable.GetAsyncEnumerator();
while (await asyncEnumerator.MoveNextAsync(CancellationToken.None))
{
Guid primaryPartition = asyncEnumerator.Current.Key;
PartitionWrapper secondaryPartition = asyncEnumerator.Current.Value;
if (secondaryPartition == null)
{
continue;
}
JToken backupInfoToken = await GetLatestBackupAvailable(primaryPartition, "http://" + secondaryPartition.primaryCluster.address + ":" + secondaryPartition.primaryCluster.httpEndpoint);
if (backupInfoToken == null)
{
continue;
}
BackupInfo backupInfo = new BackupInfo(backupInfoToken["BackupId"].ToString(), backupInfoToken["BackupLocation"].ToString(), (DateTime)backupInfoToken["CreationTimeUtc"]);
string backupPolicy = await GetPolicy("http://" + secondaryPartition.primaryCluster.address + ":" + secondaryPartition.primaryCluster.httpEndpoint, primaryPartition);
if (backupPolicy == null)
{
continue;
}
Task <RestoreResult> task = workFlowsInProgress.TryGetValue(primaryPartition, out Task <RestoreResult> value) ? value : null;
if (task == null)
{
if (secondaryPartition.LastBackupRestored == null || DateTime.Compare(backupInfo.backupTime, secondaryPartition.LastBackupRestored.backupTime) > 0)
{
Task <RestoreResult> restoreTask = Task <RestoreResult> .Run(() => RestoreWorkFlow(backupInfoToken, backupPolicy, secondaryPartition, "http://" + secondaryPartition.secondaryCluster.address + ":" + secondaryPartition.secondaryCluster.httpEndpoint, "partitionDictionary"));
workFlowsInProgress.Add(asyncEnumerator.Current.Key, restoreTask);
PartitionWrapper updatedPartitionWrapper = ObjectExtensions.Copy(secondaryPartition);
updatedPartitionWrapper.LatestBackupAvailable = backupInfo;
updatedPartitionWrapper.CurrentlyUnderRestore = backupInfo;
await myDictionary.SetAsync(tx, primaryPartition, updatedPartitionWrapper);
}
else
{
continue;
}
}
else if (task.IsCompleted)
{
RestoreResult restoreResult = task.Result;
if (restoreResult.restoreState.Equals("Success"))
{
PartitionWrapper updatedPartitionWrapper = ObjectExtensions.Copy(secondaryPartition);
updatedPartitionWrapper.LastBackupRestored = restoreResult.restoreInfo;
updatedPartitionWrapper.CurrentlyUnderRestore = null;
await myDictionary.SetAsync(tx, primaryPartition, updatedPartitionWrapper);
ServiceEventSource.Current.ServiceMessage(this.Context, "Successfully Restored!!! ");
}
workFlowsInProgress.Remove(primaryPartition);
if (secondaryPartition.LastBackupRestored == null || DateTime.Compare(backupInfo.backupTime, secondaryPartition.LastBackupRestored.backupTime) > 0)
{
Task <RestoreResult> restoreTask = Task <string> .Run(() => RestoreWorkFlow(backupInfoToken, backupPolicy, secondaryPartition, "http://" + secondaryPartition.secondaryCluster.address + ":" + secondaryPartition.secondaryCluster.httpEndpoint, "partitionDictionary"));
workFlowsInProgress.Add(primaryPartition, restoreTask);
PartitionWrapper updatedPartitionWrapper = ObjectExtensions.Copy(secondaryPartition);
updatedPartitionWrapper.LatestBackupAvailable = backupInfo;
updatedPartitionWrapper.CurrentlyUnderRestore = backupInfo;
await myDictionary.SetAsync(tx, primaryPartition, updatedPartitionWrapper);
}
else
{
continue;
}
}
else
{
PartitionWrapper updatedPartitionWrapper = ObjectExtensions.Copy(secondaryPartition);
updatedPartitionWrapper.LatestBackupAvailable = backupInfo;
await myDictionary.SetAsync(tx, primaryPartition, updatedPartitionWrapper);
}
}
await tx.CommitAsync();
}
}
// This encapsulates a restore task which triggers the restore and returns the restore result accordingly.
public async Task <RestoreResult> RestoreWorkFlow(JToken latestbackupInfo, string policy, PartitionWrapper partition, String clusterConnectionString, String partitionDictionary)
{
IReliableDictionary <Guid, PartitionWrapper> myDictionary = await this.StateManager.GetOrAddAsync <IReliableDictionary <Guid, PartitionWrapper> >(partitionDictionary);
HttpClient client = new HttpClient();
string URL = clusterConnectionString + "/Partitions/" + partition.partitionId + "/$/Restore";
string urlParameters = "?api-version=6.2-preview";
client.BaseAddress = new Uri(URL);
client.DefaultRequestHeaders.Accept.Add(
new MediaTypeWithQualityHeaderValue("application/json"));
BackupStorage backupStorage = await GetBackupStorageDetails(policy);
if (backupStorage == null)
{
ServiceEventSource.Current.ServiceMessage(this.Context, "backupstorage is null");
return(null);
}
BackupInfo backupInfo = new BackupInfo(latestbackupInfo["BackupId"].ToString(), latestbackupInfo["BackupLocation"].ToString(), backupStorage, (DateTime)latestbackupInfo["CreationTimeUtc"]);
HttpResponseMessage response = await client.PostAsJsonAsync(urlParameters, backupInfo); // Blocking call!
if (response.IsSuccessStatusCode)
{
string restoreState = "";
do
{
await Task.Delay(10000);
restoreState = GetRestoreState(partition, clusterConnectionString);
} while (restoreState.Equals("Accepted") || restoreState.Equals("RestoreInProgress"));
return(new RestoreResult(backupInfo, restoreState));
}
else
{
Console.WriteLine("{0} ({1})", (int)response.StatusCode, response.ReasonPhrase);
return(null);
}
}
// This encapsulates a restore task which triggers the restore and returns the restore result accordingly.
public async Task <RestoreResult> RestoreWorkFlow(JToken latestbackupInfo, string policy, PartitionWrapper partition, String clusterConnectionString, String clusterThumbprint, String partitionDictionary)
{
IReliableDictionary <Guid, PartitionWrapper> myDictionary = await this.StateManager.GetOrAddAsync <IReliableDictionary <Guid, PartitionWrapper> >(partitionDictionary);
string URL = clusterConnectionString + "/";
string urlParameters = "Partitions/" + partition.partitionId + "/$/Restore" + "?api-version=6.2-preview";
X509Certificate2 clientCert = GetClientCertificate(clusterThumbprint);
WebRequestHandler requestHandler = new WebRequestHandler();
requestHandler.ClientCertificates.Add(clientCert);
requestHandler.ServerCertificateValidationCallback = this.MyRemoteCertificateValidationCallback;
HttpClient client = new HttpClient(requestHandler)
{
BaseAddress = new Uri(URL)
};
client.DefaultRequestHeaders.Accept.Add(
new MediaTypeWithQualityHeaderValue("application/json"));
BackupStorage backupStorage = await GetBackupStorageDetails(policy);
if (backupStorage == null)
{
ServiceEventSource.Current.ServiceMessage(this.Context, "backupstorage is null");
return(null);
}
BackupInfo backupInfo = new BackupInfo(latestbackupInfo["BackupId"].ToString(), latestbackupInfo["BackupLocation"].ToString(), backupStorage, (DateTime)latestbackupInfo["CreationTimeUtc"]);
HttpResponseMessage response = await client.PostAsJsonAsync(urlParameters, backupInfo); // Blocking call!
if (response.IsSuccessStatusCode || response.StatusCode == System.Net.HttpStatusCode.Conflict) // As calling Restore multiple times results in 409/Conflict Error, when in progress
{
string restoreState = "";
//do
//{
// await Task.Delay(10000);
// restoreState = GetRestoreState(partition, clusterConnectionString, clusterThumbprint);
//} while (restoreState.Equals("Accepted") || restoreState.Equals("RestoreInProgress"));
restoreState = GetRestoreState(partition, clusterConnectionString, clusterThumbprint);
return(new RestoreResult(backupInfo, restoreState));
}
else
{
Console.WriteLine("{0} ({1})", (int)response.StatusCode, response.ReasonPhrase);
return(null);
}
}
// This encapsulates a restore task which triggers the restore and returns the restore result accordingly.
public async Task <RestoreResult> RestoreWorkFlow(JToken latestbackupInfo, string policy, PartitionWrapper partition, String clusterConnectionString, String clusterThumbprint)
{
string URL = clusterConnectionString + "/";
string URLParameters = "Partitions/" + partition.partitionId + "/$/Restore" + "?api-version=6.4";
BackupStorage backupStorage = await GetBackupStorageDetails(policy);
if (backupStorage == null)
{
ServiceEventSource.Current.ServiceMessage(this.Context, "backupstorage is null");
return(null);
}
BackupInfo backupInfo = new BackupInfo(latestbackupInfo["BackupId"].ToString(), latestbackupInfo["BackupLocation"].ToString(), backupStorage, (DateTime)latestbackupInfo["CreationTimeUtc"]);
HttpResponseMessage response = await Utility.HTTPPostAsync(URL, URLParameters, backupInfo, clusterThumbprint);
if (response.IsSuccessStatusCode || response.StatusCode == System.Net.HttpStatusCode.Conflict) // As calling Restore multiple times results in 409/Conflict Error, when in progress
{
string restoreState = "";
restoreState = GetRestoreState(partition, clusterConnectionString, clusterThumbprint);
return(new RestoreResult(backupInfo, restoreState));
}
else
{
Console.WriteLine("{0} ({1})", (int)response.StatusCode, response.ReasonPhrase);
return(null);
}
}