private ReplicatedTableQuorumWriteResult UpdateConfigurationInternal(ReplicatedTableConfiguration configuration, bool useConditionalUpdate) { SanitizeConfiguration(configuration); // - Upload configuration ... Func <ReplicatedTableConfiguration, ReplicatedTableConfiguration, bool> comparer = (a, b) => a.Id == b.Id; if (!useConditionalUpdate) { comparer = (a, b) => true; } ReplicatedTableQuorumWriteResult result = CloudBlobHelpers.TryWriteBlobQuorum( this.configManager.GetBlobs(), configuration, ReplicatedTableConfiguration.FromJson, comparer, ReplicatedTableConfiguration.GenerateNewConfigId); if (result.Code == ReplicatedTableQuorumWriteCode.Success) { this.configManager.Invalidate(); } else { ReplicatedTableLogger.LogError("Failed to update configuration, \n{0}", result.ToString()); } return(result); }
public void TurnReplicaOff(string storageAccountName) { if (string.IsNullOrEmpty(storageAccountName)) { throw new ArgumentNullException("storageAccountName"); } ReplicatedTableConfiguration configuration = null; // - Retrieve configuration ... ReplicatedTableQuorumReadResult readResult = RetrieveConfiguration(out configuration); if (readResult.Code != ReplicatedTableQuorumReadCode.Success) { var msg = string.Format("TurnReplicaOff={0}: failed to read configuration, \n{1}", storageAccountName, readResult.ToString()); ReplicatedTableLogger.LogError(msg); throw new Exception(msg); } // - Update all views ... configuration.TurnReplicaOff(storageAccountName); // - Write back configuration ... ReplicatedTableQuorumWriteResult writeResult = UpdateConfigurationInternal(configuration, true); if (writeResult.Code != ReplicatedTableQuorumWriteCode.Success) { var msg = string.Format("TurnReplicaOff={0}: failed to update configuration, \n{1}", storageAccountName, writeResult.ToString()); ReplicatedTableLogger.LogError(msg); throw new Exception(msg); } }
public ReplicatedTableQuorumWriteResult UpdateConfiguration(ReplicatedTableConfiguration configuration, bool useConditionalUpdate = true) { if (configuration == null) { throw new ArgumentNullException("configuration"); } return(UpdateConfigurationInternal(configuration, useConditionalUpdate)); }
public override bool Equals(object obj) { ReplicatedTableConfiguration other = obj as ReplicatedTableConfiguration; if (other == null) { return(false); } return(Id == other.Id); }
private void SanitizeConfiguration(ReplicatedTableConfiguration configuration) { foreach (var view in configuration.viewMap) { var viewName = view.Key; var viewConf = view.Value; View currentView = GetView(viewName); viewConf.SanitizeWithCurrentView(currentView); } }
public static ReplicatedTableConfiguration MakeCopy(ReplicatedTableConfiguration config) { if (config == null) { return(null); } var str = config.ToJson(); return(JsonStore <ReplicatedTableConfiguration> .Deserialize(str)); }
public static ReplicatedTableConfiguration GenerateNewConfigId(ReplicatedTableConfiguration config) { if (config == null) { throw new ArgumentNullException("config"); } var copy = MakeCopy(config); copy.Id = Guid.NewGuid(); return copy; }
public void TurnReplicaOff(string storageAccountName) { if (string.IsNullOrEmpty(storageAccountName)) { throw new ArgumentNullException("storageAccountName"); } ReplicatedTableConfiguration configuration = null; // - Retrieve configuration ... ReplicatedTableQuorumReadResult readResult = RetrieveConfiguration(out configuration); if (readResult.Code != ReplicatedTableQuorumReadCode.Success) { var msg = string.Format("TurnReplicaOff={0}: failed to read configuration, \n{1}", storageAccountName, readResult.ToString()); ReplicatedTableLogger.LogError(msg); throw new Exception(msg); } // - Parse/Update all views ... foreach (var viewConf in configuration.viewMap.Values) { var foundReplicas = viewConf.GetCurrentReplicaChain() .FindAll(r => r.StorageAccountName == storageAccountName); if (!foundReplicas.Any()) { continue; } foreach (var replica in foundReplicas) { replica.Status = ReplicaStatus.None; replica.ViewWhenTurnedOff = viewConf.ViewId; } // Update view id viewConf.ViewId++; } // - Write back configuration ... ReplicatedTableQuorumWriteResult writeResult = UpdateConfigurationInternal(configuration, true); if (writeResult.Code != ReplicatedTableQuorumWriteCode.Success) { var msg = string.Format("TurnReplicaOff={0}: failed to update configuration, \n{1}", storageAccountName, writeResult.ToString()); ReplicatedTableLogger.LogError(msg); throw new Exception(msg); } }
public static ReplicatedTableConfiguration GenerateNewConfigId(ReplicatedTableConfiguration config) { if (config == null) { throw new ArgumentNullException("config"); } var copy = MakeCopy(config); copy.Id = Guid.NewGuid(); return(copy); }
/* * Configuration management APIs */ public ReplicatedTableQuorumReadResult RetrieveConfiguration(out ReplicatedTableConfiguration configuration) { List <string> eTags; ReplicatedTableQuorumReadResult result = CloudBlobHelpers.TryReadBlobQuorum( this.configManager.GetBlobs(), out configuration, out eTags, ReplicatedTableConfiguration.FromJson); if (result.Code != ReplicatedTableQuorumReadCode.Success) { ReplicatedTableLogger.LogError("Failed to read configuration, \n{0}", result.ToString()); } return(result); }
private void SaveConfigAndRefreshItsIdAndValidateIsLoaded(ReplicatedTableConfiguration configuration, string iteration, bool validateConfigIsLoaded = true) { string msg = ""; ReplicatedTableQuorumWriteResult writeResult = UpdateConfigurationInternal(configuration, true); if (writeResult.Code != ReplicatedTableQuorumWriteCode.Success) { msg = string.Format("{0} : Failed to update configuration, \n{1}", iteration, writeResult.ToString()); ReplicatedTableLogger.LogError(msg); throw new Exception(msg); } // Update config with new Id configuration.Id = new Guid(writeResult.Message); // do we need to validate if the config is loaded by the config manager ? if (!validateConfigIsLoaded) { return; } // - Confirm the new config is the current loaded into the RTable config manager if (configuration.Id == this.configManager.GetCurrentRunningConfigId()) { return; } msg = string.Format("{0} : ConfigId({1}) != currently running configurationId({2})", iteration, configuration.Id, this.configManager.GetCurrentRunningConfigId()); ReplicatedTableLogger.LogError(msg); throw new Exception(msg); }
public ReplicatedTableQuorumWriteResult UploadConfigurationToBlobs(List <int> blobIndexes, ReplicatedTableConfiguration configuration) { if (blobIndexes == null || !blobIndexes.Any()) { throw new ArgumentNullException("blobIndexes"); } if (configuration == null) { throw new ArgumentNullException("configuration"); } List <CloudBlockBlob> blobs = new List <CloudBlockBlob>(); foreach (var blobIndex in blobIndexes) { if (blobIndex < this.configManager.GetBlobs().Count) { blobs.Add(this.configManager.GetBlobs()[blobIndex]); continue; } var msg = string.Format("blobIndex={0} >= BlobCount={1}", blobIndex, this.configManager.GetBlobs().Count); ReplicatedTableLogger.LogError(msg); throw new Exception(msg); } SanitizeConfiguration(configuration); // - Upload to blobs ... ReplicatedTableQuorumWriteResult result = CloudBlobHelpers.TryUploadBlobs(blobs, configuration); this.configManager.Invalidate(); if (result.Code != ReplicatedTableQuorumWriteCode.Success) { ReplicatedTableLogger.LogError("Failed to upload configuration to blobs, \n{0}", result.ToString()); } return(result); }
public static ReplicatedTableConfiguration MakeCopy(ReplicatedTableConfiguration config) { if (config == null) { return null; } var str = config.ToJson(); return JsonStore<ReplicatedTableConfiguration>.Deserialize(str); }
public void TurnReplicaOn(string storageAccountName, List <string> tablesToRepair, out List <ReplicatedTableRepairResult> failures) { if (string.IsNullOrEmpty(storageAccountName)) { throw new ArgumentNullException("storageAccountName"); } if (tablesToRepair == null) { throw new ArgumentNullException("tablesToRepair"); } ReplicatedTableConfiguration configuration = null; failures = new List <ReplicatedTableRepairResult>(); // - Retrieve configuration ... ReplicatedTableQuorumReadResult readResult = RetrieveConfiguration(out configuration); if (readResult.Code != ReplicatedTableQuorumReadCode.Success) { var msg = string.Format("TurnReplicaOn={0}: failed to read configuration, \n{1}", storageAccountName, readResult.ToString()); ReplicatedTableLogger.LogError(msg); throw new Exception(msg); } /* - Phase 1: * Move the *replica* to the front and set it to None. * Make the view ReadOnly. **/ #region Phase 1 configuration.MoveReplicaToHeadAndSetViewToReadOnly(storageAccountName); // - Write back configuration, refresh its Id with the new one, // but don't validate it is loaded bcz if all views of the config are empty, the config won't be refreshed by RefreshReadAndWriteViewsFromBlobs() thread! SaveConfigAndRefreshItsIdAndValidateIsLoaded(configuration, "Phase 1", false); #endregion /** * Chain is such: [None] -> [RO] -> ... -> [RO] * or: [None] -> [None] -> ... -> [None] **/ // - Wait for L + CF to make sure no pending transaction working on old views Thread.Sleep(TimeSpan.FromSeconds(Constants.LeaseDurationInSec + Constants.ClockFactorInSec)); /* - Phase 2: * Set the *replica* (head) to WriteOnly and other active replicas to ReadWrite. * Or, in case of one replic chain * Set only the *replica* (head) to ReadWrite. **/ #region Phase 2 configuration.EnableWriteOnReplicas(storageAccountName); // - Write back configuration, refresh its Id with the new one, // and then validate it is loaded now (it has to be working since next Phase is "Repair") SaveConfigAndRefreshItsIdAndValidateIsLoaded(configuration, "Phase 2"); #endregion /** * Chain is such: [W] -> [RW] -> ... -> [RW] * or: [RW] -> [None] -> ... -> [None] **/ // To be safe: // - Wait for L + CF to make sure no pending transaction working on old views Thread.Sleep(TimeSpan.FromSeconds(Constants.LeaseDurationInSec + Constants.ClockFactorInSec)); /* - Phase 3: * Repair all tables **/ #region Phase 3 foreach (var tableName in tablesToRepair) { ReplicatedTableRepairResult result = RepairTable(tableName, storageAccountName, configuration); if (result.Code != ReplicatedTableRepairCode.Error) { ReplicatedTableLogger.LogInformational(result.ToString()); continue; } ReplicatedTableLogger.LogError(result.ToString()); // List of tables (and corresponding views) failed to repair! failures.Add(result); } #endregion /* - Phase 4: * Set the *replica* (head) to ReadWrite. **/ #region Phase 4 configuration.EnableReadWriteOnReplicas(storageAccountName, failures.Select(r => r.ViewName).ToList()); // - Write back configuration, refresh its Id with the new one, // and then validate it is loaded now (i.e. it is a working config) SaveConfigAndRefreshItsIdAndValidateIsLoaded(configuration, "Phase 4"); #endregion /** * Chain is such: [RW] -> [RW] -> ... -> [RW] if all configured table repaired * or: [W] -> [RW] -> ... -> [RW] if at least one configured table failed repair ! **/ }
private ReplicatedTableQuorumWriteResult UpdateConfigurationInternal(ReplicatedTableConfiguration configuration, bool useConditionalUpdate) { SanitizeConfiguration(configuration); // - Upload configuration ... Func<ReplicatedTableConfiguration, ReplicatedTableConfiguration, bool> comparer = (a, b) => a.Id == b.Id; if (!useConditionalUpdate) { comparer = (a, b) => true; } ReplicatedTableQuorumWriteResult result = CloudBlobHelpers.TryWriteBlobQuorum( this.configManager.GetBlobs(), configuration, ReplicatedTableConfiguration.FromJson, comparer, ReplicatedTableConfiguration.GenerateNewConfigId); if (result.Code == ReplicatedTableQuorumWriteCode.Success) { this.configManager.Invalidate(); } else { ReplicatedTableLogger.LogError("Failed to update configuration, \n{0}", result.ToString()); } return result; }
private void SaveConfigAndRefreshItsIdAndValidateIsLoaded(ReplicatedTableConfiguration configuration, string iteration) { string msg = ""; ReplicatedTableQuorumWriteResult writeResult = UpdateConfigurationInternal(configuration, true); if (writeResult.Code != ReplicatedTableQuorumWriteCode.Success) { msg = string.Format("{0} : Failed to update configuration, \n{1}", iteration, writeResult.ToString()); ReplicatedTableLogger.LogError(msg); throw new Exception(msg); } // Update config with new Id configuration.Id = new Guid(writeResult.Message); // - Confirm the new config is the current loaded into the RTable config manager if (configuration.Id == this.configManager.GetCurrentRunningConfigId()) { return; } msg = string.Format("{0} : ConfigId({1}) != currently running configurationId({2})", iteration, configuration.Id, this.configManager.GetCurrentRunningConfigId()); ReplicatedTableLogger.LogError(msg); throw new Exception(msg); }
private void SanitizeConfiguration(ReplicatedTableConfiguration configuration) { foreach (var view in configuration.viewMap) { var viewName = view.Key; var viewConf = view.Value; long viewId = viewConf.ViewId; View currentView = GetView(viewName); if (viewId == 0) { if (!currentView.IsEmpty) { viewId = currentView.ViewId + 1; } else { viewId = 1; } } viewConf.Timestamp = DateTime.UtcNow; viewConf.ViewId = viewId; foreach (var replica in viewConf.GetCurrentReplicaChain()) { // We are introducing 1 or more replicas at the head. // For each such replica, update the view id in which it was added to the write view of the chain if (replica.IsWriteOnly()) { replica.ViewInWhichAddedToChain = viewId; continue; } // stop at the first Readable replica break; } } }
private ReplicatedTableRepairResult RepairTable(string tableName, string storageAccountName, ReplicatedTableConfiguration configuration) { string viewName = ""; try { ReplicatedTableConfiguredTable tableConfig; if (!configuration.IsConfiguredTable(tableName, out tableConfig)) { return new ReplicatedTableRepairResult(ReplicatedTableRepairCode.NotConfiguredTable, tableName); } viewName = tableConfig.ViewName; List<ReplicaInfo> list = configuration.GetView(viewName).ReplicaChain; if (!list.Any() || list[0].StorageAccountName != storageAccountName) { return new ReplicatedTableRepairResult(ReplicatedTableRepairCode.NotImpactedTable, tableName, viewName, storageAccountName); } ReplicaInfo head = list[0]; if (head.Status != ReplicaStatus.WriteOnly) { return new ReplicatedTableRepairResult(ReplicatedTableRepairCode.StableTable, tableName, viewName, storageAccountName); } int viewIdToRecoverFrom = (int)head.ViewWhenTurnedOff; ReplicatedTableLogger.LogInformational("RepairTable={0}, View={1}, StorageAccountName={2}, from viewId={3} ...", tableName, viewName, storageAccountName, viewIdToRecoverFrom); // Repairing ... ReconfigurationStatus status = new ReplicatedTable(tableName, this).RepairTable(viewIdToRecoverFrom, null); ReplicatedTableLogger.LogInformational("RepairTable={0}, View={1}, StorageAccountName={2}, from viewId={3} => Status={4}", tableName, viewName, storageAccountName, viewIdToRecoverFrom, status); if (status == ReconfigurationStatus.SUCCESS) { return new ReplicatedTableRepairResult(ReplicatedTableRepairCode.Success, tableName, viewName, storageAccountName); } // Failure! return new ReplicatedTableRepairResult(ReplicatedTableRepairCode.Error, tableName, viewName, storageAccountName) { Status = status, }; } catch (Exception ex) { return new ReplicatedTableRepairResult(ReplicatedTableRepairCode.Error, tableName, viewName, storageAccountName) { Status = ReconfigurationStatus.FAILURE, Message = ex.ToString(), }; } }
public ReplicatedTableQuorumWriteResult UploadConfigurationToBlobs(List<int> blobIndexes, ReplicatedTableConfiguration configuration) { if (blobIndexes == null || !blobIndexes.Any()) { throw new ArgumentNullException("blobIndexes"); } if (configuration == null) { throw new ArgumentNullException("configuration"); } List<CloudBlockBlob> blobs = new List<CloudBlockBlob>(); foreach (var blobIndex in blobIndexes) { if (blobIndex < this.configManager.GetBlobs().Count) { blobs.Add(this.configManager.GetBlobs()[blobIndex]); continue; } var msg = string.Format("blobIndex={0} >= BlobCount={1}", blobIndex, this.configManager.GetBlobs().Count); ReplicatedTableLogger.LogError(msg); throw new Exception(msg); } SanitizeConfiguration(configuration); // - Upload to blobs ... ReplicatedTableQuorumWriteResult result = CloudBlobHelpers.TryUploadBlobs(blobs, configuration); this.configManager.Invalidate(); if (result.Code != ReplicatedTableQuorumWriteCode.Success) { ReplicatedTableLogger.LogError("Failed to upload configuration to blobs, \n{0}", result.ToString()); } return result; }
public ReplicatedTableQuorumWriteResult UpdateConfiguration(ReplicatedTableConfiguration configuration, bool useConditionalUpdate = true) { if (configuration == null) { throw new ArgumentNullException("configuration"); } return UpdateConfigurationInternal(configuration, useConditionalUpdate); }
/* * Configuration management APIs */ public ReplicatedTableQuorumReadResult RetrieveConfiguration(out ReplicatedTableConfiguration configuration) { List<string> eTags; ReplicatedTableQuorumReadResult result = CloudBlobHelpers.TryReadBlobQuorum( this.configManager.GetBlobs(), out configuration, out eTags, ReplicatedTableConfiguration.FromJson); if (result.Code != ReplicatedTableQuorumReadCode.Success) { ReplicatedTableLogger.LogError("Failed to read configuration, \n{0}", result.ToString()); } return result; }
private ReplicatedTableRepairResult RepairTable(string tableName, string storageAccountName, ReplicatedTableConfiguration configuration) { string viewName = ""; try { ReplicatedTableConfiguredTable tableConfig; if (!configuration.IsConfiguredTable(tableName, out tableConfig)) { return(new ReplicatedTableRepairResult(ReplicatedTableRepairCode.NotConfiguredTable, tableName)); } viewName = tableConfig.ViewName; List <ReplicaInfo> list = configuration.GetView(viewName).ReplicaChain; if (!list.Any() || list[0].StorageAccountName != storageAccountName) { return(new ReplicatedTableRepairResult(ReplicatedTableRepairCode.NotImpactedTable, tableName, viewName, storageAccountName)); } ReplicaInfo head = list[0]; if (head.Status != ReplicaStatus.WriteOnly) { return(new ReplicatedTableRepairResult(ReplicatedTableRepairCode.StableTable, tableName, viewName, storageAccountName)); } int viewIdToRecoverFrom = (int)head.ViewWhenTurnedOff; ReplicatedTableLogger.LogInformational("RepairTable={0}, View={1}, StorageAccountName={2}, from viewId={3} ...", tableName, viewName, storageAccountName, viewIdToRecoverFrom); // Repairing ... ReconfigurationStatus status = new ReplicatedTable(tableName, this).RepairTable(viewIdToRecoverFrom, null); ReplicatedTableLogger.LogInformational("RepairTable={0}, View={1}, StorageAccountName={2}, from viewId={3} => Status={4}", tableName, viewName, storageAccountName, viewIdToRecoverFrom, status); if (status == ReconfigurationStatus.SUCCESS) { return(new ReplicatedTableRepairResult(ReplicatedTableRepairCode.Success, tableName, viewName, storageAccountName)); } // Failure! return(new ReplicatedTableRepairResult(ReplicatedTableRepairCode.Error, tableName, viewName, storageAccountName) { Status = status, }); } catch (Exception ex) { return(new ReplicatedTableRepairResult(ReplicatedTableRepairCode.Error, tableName, viewName, storageAccountName) { Status = ReconfigurationStatus.FAILURE, Message = ex.ToString(), }); } }