private ReplicatedTableQuorumWriteResult UpdateConfigurationInternal(ReplicatedTableConfiguration configuration, bool useConditionalUpdate)
{
SanitizeConfiguration(configuration);
// - Upload configuration ...
Func <ReplicatedTableConfiguration, ReplicatedTableConfiguration, bool> comparer = (a, b) => a.Id == b.Id;
if (!useConditionalUpdate)
{
comparer = (a, b) => true;
}
ReplicatedTableQuorumWriteResult
result = CloudBlobHelpers.TryWriteBlobQuorum(
this.configManager.GetBlobs(),
configuration,
ReplicatedTableConfiguration.FromJson,
comparer,
ReplicatedTableConfiguration.GenerateNewConfigId);
if (result.Code == ReplicatedTableQuorumWriteCode.Success)
{
this.configManager.Invalidate();
}
else
{
ReplicatedTableLogger.LogError("Failed to update configuration, \n{0}", result.ToString());
}
return(result);
}
public void TurnReplicaOff(string storageAccountName)
{
if (string.IsNullOrEmpty(storageAccountName))
{
throw new ArgumentNullException("storageAccountName");
}
ReplicatedTableConfiguration configuration = null;
// - Retrieve configuration ...
ReplicatedTableQuorumReadResult readResult = RetrieveConfiguration(out configuration);
if (readResult.Code != ReplicatedTableQuorumReadCode.Success)
{
var msg = string.Format("TurnReplicaOff={0}: failed to read configuration, \n{1}", storageAccountName, readResult.ToString());
ReplicatedTableLogger.LogError(msg);
throw new Exception(msg);
}
// - Update all views ...
configuration.TurnReplicaOff(storageAccountName);
// - Write back configuration ...
ReplicatedTableQuorumWriteResult writeResult = UpdateConfigurationInternal(configuration, true);
if (writeResult.Code != ReplicatedTableQuorumWriteCode.Success)
{
var msg = string.Format("TurnReplicaOff={0}: failed to update configuration, \n{1}", storageAccountName, writeResult.ToString());
ReplicatedTableLogger.LogError(msg);
throw new Exception(msg);
}
}
public ReplicatedTableQuorumWriteResult UpdateConfiguration(ReplicatedTableConfiguration configuration, bool useConditionalUpdate = true)
{
if (configuration == null)
{
throw new ArgumentNullException("configuration");
}
return(UpdateConfigurationInternal(configuration, useConditionalUpdate));
}
public override bool Equals(object obj)
{
ReplicatedTableConfiguration other = obj as ReplicatedTableConfiguration;
if (other == null)
{
return(false);
}
return(Id == other.Id);
}
private void SanitizeConfiguration(ReplicatedTableConfiguration configuration)
{
foreach (var view in configuration.viewMap)
{
var viewName = view.Key;
var viewConf = view.Value;
View currentView = GetView(viewName);
viewConf.SanitizeWithCurrentView(currentView);
}
}
public static ReplicatedTableConfiguration MakeCopy(ReplicatedTableConfiguration config)
{
if (config == null)
{
return(null);
}
var str = config.ToJson();
return(JsonStore <ReplicatedTableConfiguration> .Deserialize(str));
}
public static ReplicatedTableConfiguration GenerateNewConfigId(ReplicatedTableConfiguration config)
{
if (config == null)
{
throw new ArgumentNullException("config");
}
var copy = MakeCopy(config);
copy.Id = Guid.NewGuid();
return copy;
}
public void TurnReplicaOff(string storageAccountName)
{
if (string.IsNullOrEmpty(storageAccountName))
{
throw new ArgumentNullException("storageAccountName");
}
ReplicatedTableConfiguration configuration = null;
// - Retrieve configuration ...
ReplicatedTableQuorumReadResult readResult = RetrieveConfiguration(out configuration);
if (readResult.Code != ReplicatedTableQuorumReadCode.Success)
{
var msg = string.Format("TurnReplicaOff={0}: failed to read configuration, \n{1}", storageAccountName, readResult.ToString());
ReplicatedTableLogger.LogError(msg);
throw new Exception(msg);
}
// - Parse/Update all views ...
foreach (var viewConf in configuration.viewMap.Values)
{
var foundReplicas = viewConf.GetCurrentReplicaChain()
.FindAll(r => r.StorageAccountName == storageAccountName);
if (!foundReplicas.Any())
{
continue;
}
foreach (var replica in foundReplicas)
{
replica.Status = ReplicaStatus.None;
replica.ViewWhenTurnedOff = viewConf.ViewId;
}
// Update view id
viewConf.ViewId++;
}
// - Write back configuration ...
ReplicatedTableQuorumWriteResult writeResult = UpdateConfigurationInternal(configuration, true);
if (writeResult.Code != ReplicatedTableQuorumWriteCode.Success)
{
var msg = string.Format("TurnReplicaOff={0}: failed to update configuration, \n{1}", storageAccountName, writeResult.ToString());
ReplicatedTableLogger.LogError(msg);
throw new Exception(msg);
}
}
public static ReplicatedTableConfiguration GenerateNewConfigId(ReplicatedTableConfiguration config)
{
if (config == null)
{
throw new ArgumentNullException("config");
}
var copy = MakeCopy(config);
copy.Id = Guid.NewGuid();
return(copy);
}
/*
* Configuration management APIs
*/
public ReplicatedTableQuorumReadResult RetrieveConfiguration(out ReplicatedTableConfiguration configuration)
{
List <string> eTags;
ReplicatedTableQuorumReadResult
result = CloudBlobHelpers.TryReadBlobQuorum(
this.configManager.GetBlobs(),
out configuration,
out eTags,
ReplicatedTableConfiguration.FromJson);
if (result.Code != ReplicatedTableQuorumReadCode.Success)
{
ReplicatedTableLogger.LogError("Failed to read configuration, \n{0}", result.ToString());
}
return(result);
}
private void SaveConfigAndRefreshItsIdAndValidateIsLoaded(ReplicatedTableConfiguration configuration, string iteration, bool validateConfigIsLoaded = true)
{
string msg = "";
ReplicatedTableQuorumWriteResult writeResult = UpdateConfigurationInternal(configuration, true);
if (writeResult.Code != ReplicatedTableQuorumWriteCode.Success)
{
msg = string.Format("{0} : Failed to update configuration, \n{1}", iteration, writeResult.ToString());
ReplicatedTableLogger.LogError(msg);
throw new Exception(msg);
}
// Update config with new Id
configuration.Id = new Guid(writeResult.Message);
// do we need to validate if the config is loaded by the config manager ?
if (!validateConfigIsLoaded)
{
return;
}
// - Confirm the new config is the current loaded into the RTable config manager
if (configuration.Id == this.configManager.GetCurrentRunningConfigId())
{
return;
}
msg = string.Format("{0} : ConfigId({1}) != currently running configurationId({2})",
iteration,
configuration.Id,
this.configManager.GetCurrentRunningConfigId());
ReplicatedTableLogger.LogError(msg);
throw new Exception(msg);
}
public ReplicatedTableQuorumWriteResult UploadConfigurationToBlobs(List <int> blobIndexes, ReplicatedTableConfiguration configuration)
{
if (blobIndexes == null || !blobIndexes.Any())
{
throw new ArgumentNullException("blobIndexes");
}
if (configuration == null)
{
throw new ArgumentNullException("configuration");
}
List <CloudBlockBlob> blobs = new List <CloudBlockBlob>();
foreach (var blobIndex in blobIndexes)
{
if (blobIndex < this.configManager.GetBlobs().Count)
{
blobs.Add(this.configManager.GetBlobs()[blobIndex]);
continue;
}
var msg = string.Format("blobIndex={0} >= BlobCount={1}", blobIndex, this.configManager.GetBlobs().Count);
ReplicatedTableLogger.LogError(msg);
throw new Exception(msg);
}
SanitizeConfiguration(configuration);
// - Upload to blobs ...
ReplicatedTableQuorumWriteResult result = CloudBlobHelpers.TryUploadBlobs(blobs, configuration);
this.configManager.Invalidate();
if (result.Code != ReplicatedTableQuorumWriteCode.Success)
{
ReplicatedTableLogger.LogError("Failed to upload configuration to blobs, \n{0}", result.ToString());
}
return(result);
}
public static ReplicatedTableConfiguration MakeCopy(ReplicatedTableConfiguration config)
{
if (config == null)
{
return null;
}
var str = config.ToJson();
return JsonStore<ReplicatedTableConfiguration>.Deserialize(str);
}
public void TurnReplicaOn(string storageAccountName, List <string> tablesToRepair, out List <ReplicatedTableRepairResult> failures)
{
if (string.IsNullOrEmpty(storageAccountName))
{
throw new ArgumentNullException("storageAccountName");
}
if (tablesToRepair == null)
{
throw new ArgumentNullException("tablesToRepair");
}
ReplicatedTableConfiguration configuration = null;
failures = new List <ReplicatedTableRepairResult>();
// - Retrieve configuration ...
ReplicatedTableQuorumReadResult readResult = RetrieveConfiguration(out configuration);
if (readResult.Code != ReplicatedTableQuorumReadCode.Success)
{
var msg = string.Format("TurnReplicaOn={0}: failed to read configuration, \n{1}", storageAccountName, readResult.ToString());
ReplicatedTableLogger.LogError(msg);
throw new Exception(msg);
}
/* - Phase 1:
* Move the *replica* to the front and set it to None.
* Make the view ReadOnly.
**/
#region Phase 1
configuration.MoveReplicaToHeadAndSetViewToReadOnly(storageAccountName);
// - Write back configuration, refresh its Id with the new one,
// but don't validate it is loaded bcz if all views of the config are empty, the config won't be refreshed by RefreshReadAndWriteViewsFromBlobs() thread!
SaveConfigAndRefreshItsIdAndValidateIsLoaded(configuration, "Phase 1", false);
#endregion
/**
* Chain is such: [None] -> [RO] -> ... -> [RO]
* or: [None] -> [None] -> ... -> [None]
**/
// - Wait for L + CF to make sure no pending transaction working on old views
Thread.Sleep(TimeSpan.FromSeconds(Constants.LeaseDurationInSec + Constants.ClockFactorInSec));
/* - Phase 2:
* Set the *replica* (head) to WriteOnly and other active replicas to ReadWrite.
* Or, in case of one replic chain
* Set only the *replica* (head) to ReadWrite.
**/
#region Phase 2
configuration.EnableWriteOnReplicas(storageAccountName);
// - Write back configuration, refresh its Id with the new one,
// and then validate it is loaded now (it has to be working since next Phase is "Repair")
SaveConfigAndRefreshItsIdAndValidateIsLoaded(configuration, "Phase 2");
#endregion
/**
* Chain is such: [W] -> [RW] -> ... -> [RW]
* or: [RW] -> [None] -> ... -> [None]
**/
// To be safe:
// - Wait for L + CF to make sure no pending transaction working on old views
Thread.Sleep(TimeSpan.FromSeconds(Constants.LeaseDurationInSec + Constants.ClockFactorInSec));
/* - Phase 3:
* Repair all tables
**/
#region Phase 3
foreach (var tableName in tablesToRepair)
{
ReplicatedTableRepairResult result = RepairTable(tableName, storageAccountName, configuration);
if (result.Code != ReplicatedTableRepairCode.Error)
{
ReplicatedTableLogger.LogInformational(result.ToString());
continue;
}
ReplicatedTableLogger.LogError(result.ToString());
// List of tables (and corresponding views) failed to repair!
failures.Add(result);
}
#endregion
/* - Phase 4:
* Set the *replica* (head) to ReadWrite.
**/
#region Phase 4
configuration.EnableReadWriteOnReplicas(storageAccountName, failures.Select(r => r.ViewName).ToList());
// - Write back configuration, refresh its Id with the new one,
// and then validate it is loaded now (i.e. it is a working config)
SaveConfigAndRefreshItsIdAndValidateIsLoaded(configuration, "Phase 4");
#endregion
/**
* Chain is such: [RW] -> [RW] -> ... -> [RW] if all configured table repaired
* or: [W] -> [RW] -> ... -> [RW] if at least one configured table failed repair !
**/
}
private ReplicatedTableQuorumWriteResult UpdateConfigurationInternal(ReplicatedTableConfiguration configuration, bool useConditionalUpdate)
{
SanitizeConfiguration(configuration);
// - Upload configuration ...
Func<ReplicatedTableConfiguration, ReplicatedTableConfiguration, bool> comparer = (a, b) => a.Id == b.Id;
if (!useConditionalUpdate)
{
comparer = (a, b) => true;
}
ReplicatedTableQuorumWriteResult
result = CloudBlobHelpers.TryWriteBlobQuorum(
this.configManager.GetBlobs(),
configuration,
ReplicatedTableConfiguration.FromJson,
comparer,
ReplicatedTableConfiguration.GenerateNewConfigId);
if (result.Code == ReplicatedTableQuorumWriteCode.Success)
{
this.configManager.Invalidate();
}
else
{
ReplicatedTableLogger.LogError("Failed to update configuration, \n{0}", result.ToString());
}
return result;
}
private void SaveConfigAndRefreshItsIdAndValidateIsLoaded(ReplicatedTableConfiguration configuration, string iteration)
{
string msg = "";
ReplicatedTableQuorumWriteResult writeResult = UpdateConfigurationInternal(configuration, true);
if (writeResult.Code != ReplicatedTableQuorumWriteCode.Success)
{
msg = string.Format("{0} : Failed to update configuration, \n{1}", iteration, writeResult.ToString());
ReplicatedTableLogger.LogError(msg);
throw new Exception(msg);
}
// Update config with new Id
configuration.Id = new Guid(writeResult.Message);
// - Confirm the new config is the current loaded into the RTable config manager
if (configuration.Id == this.configManager.GetCurrentRunningConfigId())
{
return;
}
msg = string.Format("{0} : ConfigId({1}) != currently running configurationId({2})",
iteration,
configuration.Id,
this.configManager.GetCurrentRunningConfigId());
ReplicatedTableLogger.LogError(msg);
throw new Exception(msg);
}
private void SanitizeConfiguration(ReplicatedTableConfiguration configuration)
{
foreach (var view in configuration.viewMap)
{
var viewName = view.Key;
var viewConf = view.Value;
long viewId = viewConf.ViewId;
View currentView = GetView(viewName);
if (viewId == 0)
{
if (!currentView.IsEmpty)
{
viewId = currentView.ViewId + 1;
}
else
{
viewId = 1;
}
}
viewConf.Timestamp = DateTime.UtcNow;
viewConf.ViewId = viewId;
foreach (var replica in viewConf.GetCurrentReplicaChain())
{
// We are introducing 1 or more replicas at the head.
// For each such replica, update the view id in which it was added to the write view of the chain
if (replica.IsWriteOnly())
{
replica.ViewInWhichAddedToChain = viewId;
continue;
}
// stop at the first Readable replica
break;
}
}
}
private ReplicatedTableRepairResult RepairTable(string tableName, string storageAccountName, ReplicatedTableConfiguration configuration)
{
string viewName = "";
try
{
ReplicatedTableConfiguredTable tableConfig;
if (!configuration.IsConfiguredTable(tableName, out tableConfig))
{
return new ReplicatedTableRepairResult(ReplicatedTableRepairCode.NotConfiguredTable, tableName);
}
viewName = tableConfig.ViewName;
List<ReplicaInfo> list = configuration.GetView(viewName).ReplicaChain;
if (!list.Any() ||
list[0].StorageAccountName != storageAccountName)
{
return new ReplicatedTableRepairResult(ReplicatedTableRepairCode.NotImpactedTable, tableName, viewName, storageAccountName);
}
ReplicaInfo head = list[0];
if (head.Status != ReplicaStatus.WriteOnly)
{
return new ReplicatedTableRepairResult(ReplicatedTableRepairCode.StableTable, tableName, viewName, storageAccountName);
}
int viewIdToRecoverFrom = (int)head.ViewWhenTurnedOff;
ReplicatedTableLogger.LogInformational("RepairTable={0}, View={1}, StorageAccountName={2}, from viewId={3} ...",
tableName,
viewName,
storageAccountName,
viewIdToRecoverFrom);
// Repairing ...
ReconfigurationStatus status = new ReplicatedTable(tableName, this).RepairTable(viewIdToRecoverFrom, null);
ReplicatedTableLogger.LogInformational("RepairTable={0}, View={1}, StorageAccountName={2}, from viewId={3} => Status={4}",
tableName,
viewName,
storageAccountName,
viewIdToRecoverFrom,
status);
if (status == ReconfigurationStatus.SUCCESS)
{
return new ReplicatedTableRepairResult(ReplicatedTableRepairCode.Success, tableName, viewName, storageAccountName);
}
// Failure!
return new ReplicatedTableRepairResult(ReplicatedTableRepairCode.Error, tableName, viewName, storageAccountName)
{
Status = status,
};
}
catch (Exception ex)
{
return new ReplicatedTableRepairResult(ReplicatedTableRepairCode.Error, tableName, viewName, storageAccountName)
{
Status = ReconfigurationStatus.FAILURE,
Message = ex.ToString(),
};
}
}
public ReplicatedTableQuorumWriteResult UploadConfigurationToBlobs(List<int> blobIndexes, ReplicatedTableConfiguration configuration)
{
if (blobIndexes == null || !blobIndexes.Any())
{
throw new ArgumentNullException("blobIndexes");
}
if (configuration == null)
{
throw new ArgumentNullException("configuration");
}
List<CloudBlockBlob> blobs = new List<CloudBlockBlob>();
foreach (var blobIndex in blobIndexes)
{
if (blobIndex < this.configManager.GetBlobs().Count)
{
blobs.Add(this.configManager.GetBlobs()[blobIndex]);
continue;
}
var msg = string.Format("blobIndex={0} >= BlobCount={1}", blobIndex, this.configManager.GetBlobs().Count);
ReplicatedTableLogger.LogError(msg);
throw new Exception(msg);
}
SanitizeConfiguration(configuration);
// - Upload to blobs ...
ReplicatedTableQuorumWriteResult result = CloudBlobHelpers.TryUploadBlobs(blobs, configuration);
this.configManager.Invalidate();
if (result.Code != ReplicatedTableQuorumWriteCode.Success)
{
ReplicatedTableLogger.LogError("Failed to upload configuration to blobs, \n{0}", result.ToString());
}
return result;
}
public ReplicatedTableQuorumWriteResult UpdateConfiguration(ReplicatedTableConfiguration configuration, bool useConditionalUpdate = true)
{
if (configuration == null)
{
throw new ArgumentNullException("configuration");
}
return UpdateConfigurationInternal(configuration, useConditionalUpdate);
}
/*
* Configuration management APIs
*/
public ReplicatedTableQuorumReadResult RetrieveConfiguration(out ReplicatedTableConfiguration configuration)
{
List<string> eTags;
ReplicatedTableQuorumReadResult
result = CloudBlobHelpers.TryReadBlobQuorum(
this.configManager.GetBlobs(),
out configuration,
out eTags,
ReplicatedTableConfiguration.FromJson);
if (result.Code != ReplicatedTableQuorumReadCode.Success)
{
ReplicatedTableLogger.LogError("Failed to read configuration, \n{0}", result.ToString());
}
return result;
}
private ReplicatedTableRepairResult RepairTable(string tableName, string storageAccountName, ReplicatedTableConfiguration configuration)
{
string viewName = "";
try
{
ReplicatedTableConfiguredTable tableConfig;
if (!configuration.IsConfiguredTable(tableName, out tableConfig))
{
return(new ReplicatedTableRepairResult(ReplicatedTableRepairCode.NotConfiguredTable, tableName));
}
viewName = tableConfig.ViewName;
List <ReplicaInfo> list = configuration.GetView(viewName).ReplicaChain;
if (!list.Any() ||
list[0].StorageAccountName != storageAccountName)
{
return(new ReplicatedTableRepairResult(ReplicatedTableRepairCode.NotImpactedTable, tableName, viewName, storageAccountName));
}
ReplicaInfo head = list[0];
if (head.Status != ReplicaStatus.WriteOnly)
{
return(new ReplicatedTableRepairResult(ReplicatedTableRepairCode.StableTable, tableName, viewName, storageAccountName));
}
int viewIdToRecoverFrom = (int)head.ViewWhenTurnedOff;
ReplicatedTableLogger.LogInformational("RepairTable={0}, View={1}, StorageAccountName={2}, from viewId={3} ...",
tableName,
viewName,
storageAccountName,
viewIdToRecoverFrom);
// Repairing ...
ReconfigurationStatus status = new ReplicatedTable(tableName, this).RepairTable(viewIdToRecoverFrom, null);
ReplicatedTableLogger.LogInformational("RepairTable={0}, View={1}, StorageAccountName={2}, from viewId={3} => Status={4}",
tableName,
viewName,
storageAccountName,
viewIdToRecoverFrom,
status);
if (status == ReconfigurationStatus.SUCCESS)
{
return(new ReplicatedTableRepairResult(ReplicatedTableRepairCode.Success, tableName, viewName, storageAccountName));
}
// Failure!
return(new ReplicatedTableRepairResult(ReplicatedTableRepairCode.Error, tableName, viewName, storageAccountName)
{
Status = status,
});
}
catch (Exception ex)
{
return(new ReplicatedTableRepairResult(ReplicatedTableRepairCode.Error, tableName, viewName, storageAccountName)
{
Status = ReconfigurationStatus.FAILURE,
Message = ex.ToString(),
});
}
}