public static List <ReplicatedTableReadBlobResult> TryReadAllBlobs <T>(List <CloudBlockBlob> blobs, out List <T> values, out List <string> eTags, Func <string, T> ParseBlobFunc) where T : class { int numberOfBlobs = blobs.Count; T[] valuesArray = new T[numberOfBlobs]; string[] eTagsArray = new string[numberOfBlobs]; ReplicatedTableReadBlobResult[] resultArray = new ReplicatedTableReadBlobResult[numberOfBlobs]; // read from all the blobs in parallel Parallel.For(0, numberOfBlobs, index => { DateTime startTime = DateTime.UtcNow; T currentValue; string currentETag; resultArray[index] = TryReadBlob(blobs[index], out currentValue, out currentETag, ParseBlobFunc); valuesArray[index] = currentValue; eTagsArray[index] = currentETag; ReplicatedTableLogger.LogInformational("TryReadBlob #{0} took {1}", index, DateTime.UtcNow - startTime); }); values = valuesArray.ToList(); eTags = eTagsArray.ToList(); return(resultArray.ToList()); }
public static async Task <ReplicatedTableReadBlobResult> TryReadBlobAsync <T>(CloudBlockBlob blob, Action <T, string> callback, Func <string, T> ParseBlobFunc, CancellationToken ct) where T : class { try { BlobRequestOptions options = new BlobRequestOptions() { ServerTimeout = TimeSpan.FromSeconds(5), MaximumExecutionTime = TimeSpan.FromSeconds(30) }; string content = await blob.DownloadTextAsync(null, null, options, null, ct); if (content == Constants.ConfigurationStoreUpdatingText) { return(new ReplicatedTableReadBlobResult(ReadBlobCode.UpdateInProgress, "Blob update in progress ...")); } // ParseBlobFunc != null T configuration = ParseBlobFunc(content); string eTag = blob.Properties.ETag; // callback != null callback(configuration, eTag); return(new ReplicatedTableReadBlobResult(ReadBlobCode.Success, "")); } catch (StorageException e) { var msg = string.Format("Error reading blob: {0}. StorageException: {1}", blob.Uri, e.Message); ReplicatedTableLogger.LogError(msg); if (e.RequestInformation != null && e.RequestInformation.HttpStatusCode == (int)HttpStatusCode.NotFound) { return(new ReplicatedTableReadBlobResult(ReadBlobCode.NotFound, msg)); } return(new ReplicatedTableReadBlobResult(ReadBlobCode.StorageException, msg)); } catch (OperationCanceledException) { var msg = string.Format("TryReadBlobAsync cancelled ({0})", blob.Uri); ReplicatedTableLogger.LogInformational(msg); return(new ReplicatedTableReadBlobResult(ReadBlobCode.Exception, msg)); } catch (Exception e) { var msg = string.Format("Error reading blob: {0}. Exception: {1}", blob.Uri, e.Message); ReplicatedTableLogger.LogError(msg); return(new ReplicatedTableReadBlobResult(ReadBlobCode.Exception, msg)); } }
private bool DoesViewNeedRefresh() { lock (this) { if ((DateTime.UtcNow - this.lastViewRefreshTime) > TimeSpan.FromSeconds(Constants.LeaseRenewalIntervalInSec)) { ReplicatedTableLogger.LogInformational("Need to renew lease on the view/refresh the view"); return(true); } return(false); } }
public static ReplicatedTableQuorumReadResult TryReadBlobQuorumFast <T>(List <CloudBlockBlob> blobs, out T value, out List <string> eTags, Func <string, T> ParseBlobFunc) where T : class { value = default(T); eTags = null; int numberOfBlobs = blobs.Count; var valuesArray = new List <T>(new T[numberOfBlobs]); var eTagsArray = new List <string>(new string[numberOfBlobs]); var resultArray = new List <ReplicatedTableReadBlobResult>(new ReplicatedTableReadBlobResult[numberOfBlobs]); var cancel = new CancellationTokenSource(); /* * Read async all the blobs in parallel */ Parallel.For(0, numberOfBlobs, async(index) => { valuesArray[index] = default(T); eTagsArray[index] = null; resultArray[index] = new ReplicatedTableReadBlobResult(ReadBlobCode.NullObject, "downloaded not started yet!"); DateTime startTime = DateTime.UtcNow; resultArray[index] = await TryReadBlobAsync( blobs[index], (currentValue, currentETag) => { valuesArray[index] = currentValue; eTagsArray[index] = currentETag; }, ParseBlobFunc, cancel.Token); if (resultArray[index].Code == ReadBlobCode.Success) { ReplicatedTableLogger.LogInformational("TryReadBlobAsync #{0} took {1}", index, DateTime.UtcNow - startTime); } }); /* * Poll for "Quorum" progress ... */ ReplicatedTableQuorumReadResult majority; int quorumIndex; do { Thread.Sleep(Constants.QuorumPollingInMilliSeconds); // "resultArray" may change OOB just after Evaluate Quorum step below. // In such case, we may exit the loop because all blobs are retrieved, while "majority" has a stale value! // Therefore, we have to determine, before Evaluate Quorum step, if we'll exit the loop or no. // If Exist, then "majority" would be final. // If No, then we'll loop to compute the latest, unless we break because we already have Quorum without all blobs. bool allBlobsRetrieved = true; // IMPORTANT: foreach()/LINQ throws when "resultArray" changes (race condition). for (int result = 0; result < resultArray.Count; result++) { if (resultArray[result].Code == ReadBlobCode.NullObject) { allBlobsRetrieved = false; break; } } // Evaluate Quorum ... majority = FindMajority(resultArray.AsReadOnly(), valuesArray.AsReadOnly(), out quorumIndex); if (majority.Code == ReplicatedTableQuorumReadCode.NotFound || majority.Code == ReplicatedTableQuorumReadCode.UpdateInProgress || majority.Code == ReplicatedTableQuorumReadCode.Exception || majority.Code == ReplicatedTableQuorumReadCode.Success) { // Quorum => cancel tasks and exit cancel.Cancel(); break; } //else if (allBlobsRetrieved) { // All blobs 'were' retrieved => exit break; } // keep polling ... } while (true); cancel = null; if (majority.Code == ReplicatedTableQuorumReadCode.Success) { value = valuesArray[quorumIndex]; eTags = eTagsArray; } return(majority); }
private ReplicatedTableRepairResult RepairTable(string tableName, string storageAccountName, ReplicatedTableConfiguration configuration) { string viewName = ""; try { ReplicatedTableConfiguredTable tableConfig; if (!configuration.IsConfiguredTable(tableName, out tableConfig)) { return(new ReplicatedTableRepairResult(ReplicatedTableRepairCode.NotConfiguredTable, tableName)); } viewName = tableConfig.ViewName; List <ReplicaInfo> list = configuration.GetView(viewName).ReplicaChain; if (!list.Any() || list[0].StorageAccountName != storageAccountName) { return(new ReplicatedTableRepairResult(ReplicatedTableRepairCode.NotImpactedTable, tableName, viewName, storageAccountName)); } ReplicaInfo head = list[0]; if (head.Status != ReplicaStatus.WriteOnly) { return(new ReplicatedTableRepairResult(ReplicatedTableRepairCode.StableTable, tableName, viewName, storageAccountName)); } int viewIdToRecoverFrom = (int)head.ViewWhenTurnedOff; ReplicatedTableLogger.LogInformational("RepairTable={0}, View={1}, StorageAccountName={2}, from viewId={3} ...", tableName, viewName, storageAccountName, viewIdToRecoverFrom); // Repairing ... ReconfigurationStatus status = new ReplicatedTable(tableName, this).RepairTable(viewIdToRecoverFrom, null); ReplicatedTableLogger.LogInformational("RepairTable={0}, View={1}, StorageAccountName={2}, from viewId={3} => Status={4}", tableName, viewName, storageAccountName, viewIdToRecoverFrom, status); if (status == ReconfigurationStatus.SUCCESS) { return(new ReplicatedTableRepairResult(ReplicatedTableRepairCode.Success, tableName, viewName, storageAccountName)); } // Failure! return(new ReplicatedTableRepairResult(ReplicatedTableRepairCode.Error, tableName, viewName, storageAccountName) { Status = status, }); } catch (Exception ex) { return(new ReplicatedTableRepairResult(ReplicatedTableRepairCode.Error, tableName, viewName, storageAccountName) { Status = ReconfigurationStatus.FAILURE, Message = ex.ToString(), }); } }
public void TurnReplicaOn(string storageAccountName, List <string> tablesToRepair, out List <ReplicatedTableRepairResult> failures) { if (string.IsNullOrEmpty(storageAccountName)) { throw new ArgumentNullException("storageAccountName"); } if (tablesToRepair == null) { throw new ArgumentNullException("tablesToRepair"); } ReplicatedTableConfiguration configuration = null; failures = new List <ReplicatedTableRepairResult>(); // - Retrieve configuration ... ReplicatedTableQuorumReadResult readResult = RetrieveConfiguration(out configuration); if (readResult.Code != ReplicatedTableQuorumReadCode.Success) { var msg = string.Format("TurnReplicaOn={0}: failed to read configuration, \n{1}", storageAccountName, readResult.ToString()); ReplicatedTableLogger.LogError(msg); throw new Exception(msg); } /* - Phase 1: * Move the *replica* to the front and set it to None. * Make the view ReadOnly. **/ #region Phase 1 configuration.MoveReplicaToHeadAndSetViewToReadOnly(storageAccountName); // - Write back configuration, refresh its Id with the new one, // but don't validate it is loaded bcz if all views of the config are empty, the config won't be refreshed by RefreshReadAndWriteViewsFromBlobs() thread! SaveConfigAndRefreshItsIdAndValidateIsLoaded(configuration, "Phase 1", false); #endregion /** * Chain is such: [None] -> [RO] -> ... -> [RO] * or: [None] -> [None] -> ... -> [None] **/ // - Wait for L + CF to make sure no pending transaction working on old views Thread.Sleep(TimeSpan.FromSeconds(Constants.LeaseDurationInSec + Constants.ClockFactorInSec)); /* - Phase 2: * Set the *replica* (head) to WriteOnly and other active replicas to ReadWrite. * Or, in case of one replic chain * Set only the *replica* (head) to ReadWrite. **/ #region Phase 2 configuration.EnableWriteOnReplicas(storageAccountName); // - Write back configuration, refresh its Id with the new one, // and then validate it is loaded now (it has to be working since next Phase is "Repair") SaveConfigAndRefreshItsIdAndValidateIsLoaded(configuration, "Phase 2"); #endregion /** * Chain is such: [W] -> [RW] -> ... -> [RW] * or: [RW] -> [None] -> ... -> [None] **/ // To be safe: // - Wait for L + CF to make sure no pending transaction working on old views Thread.Sleep(TimeSpan.FromSeconds(Constants.LeaseDurationInSec + Constants.ClockFactorInSec)); /* - Phase 3: * Repair all tables **/ #region Phase 3 foreach (var tableName in tablesToRepair) { ReplicatedTableRepairResult result = RepairTable(tableName, storageAccountName, configuration); if (result.Code != ReplicatedTableRepairCode.Error) { ReplicatedTableLogger.LogInformational(result.ToString()); continue; } ReplicatedTableLogger.LogError(result.ToString()); // List of tables (and corresponding views) failed to repair! failures.Add(result); } #endregion /* - Phase 4: * Set the *replica* (head) to ReadWrite. **/ #region Phase 4 configuration.EnableReadWriteOnReplicas(storageAccountName, failures.Select(r => r.ViewName).ToList()); // - Write back configuration, refresh its Id with the new one, // and then validate it is loaded now (i.e. it is a working config) SaveConfigAndRefreshItsIdAndValidateIsLoaded(configuration, "Phase 4"); #endregion /** * Chain is such: [RW] -> [RW] -> ... -> [RW] if all configured table repaired * or: [W] -> [RW] -> ... -> [RW] if at least one configured table failed repair ! **/ }
private void RefreshReadAndWriteViewsFromBlobs(object arg) { List <ReplicatedTableConfiguredTable> tableConfigList; int leaseDuration; Guid configId; List <View> views; bool instrumentationFlag; bool ignoreHigherViewIdRows; // Lock because both SetConnectionStringStrategy and connectionStringMap can be updated OOB! lock (connectionStringLock) { DateTime startTime = DateTime.UtcNow; views = this.blobParser.ParseBlob( this.blobs.Values.ToList(), this.SetConnectionStringStrategy, out tableConfigList, out leaseDuration, out configId, out instrumentationFlag, out ignoreHigherViewIdRows); ReplicatedTableLogger.LogInformational("ParseBlob took {0}", DateTime.UtcNow - startTime); if (views == null) { return; } } lock (this) { // - Update lease duration LeaseDuration = TimeSpan.FromSeconds(leaseDuration); // - Update list of views this.viewMap.Clear(); foreach (var view in views) { if (view == null || string.IsNullOrEmpty(view.Name)) { continue; } // Set view LeaseDuration to the config LeaseDuration view.LeaseDuration = LeaseDuration; this.viewMap.Add(view.Name, view); } // - Update list of configured tables this.tableMap.Clear(); defaultConfiguredRule = null; if (tableConfigList != null) { foreach (var tableConfig in tableConfigList) { if (tableConfig == null || string.IsNullOrEmpty(tableConfig.TableName)) { continue; } this.tableMap.Add(tableConfig.TableName, tableConfig); if (tableConfig.UseAsDefault) { defaultConfiguredRule = tableConfig; } } } // - Update current config Id currentRunningConfigId = configId; // update instrumentation flag this.instrumentation = instrumentationFlag; // update ignoreHigherViewIdRows flag this.ignoreHigherViewIdRows = ignoreHigherViewIdRows; UpdateTimer(); } }
private void RefreshReadAndWriteViewsFromBlobs(object arg) { lock (this) { this.lastRenewedReadView = new View(); this.lastRenewedWriteView = new View(); DateTime refreshStartTime = DateTime.UtcNow; Dictionary <long, List <CloudBlockBlob> > viewResult = new Dictionary <long, List <CloudBlockBlob> >(); foreach (var blob in this.blobs) { ReplicatedTableConfigurationStore configurationStore; if (!CloudBlobHelpers.TryReadBlob <ReplicatedTableConfigurationStore>(blob.Value, out configurationStore)) { continue; } if (configurationStore.ViewId <= 0) { ReplicatedTableLogger.LogInformational("ViewId={0} is invalid. Must be >= 1. Skipping this blob {1}.", configurationStore.ViewId, blob.Value.Uri); continue; } List <CloudBlockBlob> viewBlobs; if (!viewResult.TryGetValue(configurationStore.ViewId, out viewBlobs)) { viewBlobs = new List <CloudBlockBlob>(); viewResult.Add(configurationStore.ViewId, viewBlobs); } viewBlobs.Add(blob.Value); if (viewBlobs.Count >= this.quorumSize) { this.lastRenewedReadView.ViewId = this.lastRenewedWriteView.ViewId = configurationStore.ViewId; for (int i = 0; i < configurationStore.ReplicaChain.Count; i++) { ReplicaInfo replica = configurationStore.ReplicaChain[i]; CloudTableClient tableClient = GetTableClientForReplica(replica); if (replica != null && tableClient != null) { //Update the write view always this.lastRenewedWriteView.Chain.Add(new Tuple <ReplicaInfo, CloudTableClient>(replica, tableClient)); //Update the read view only for replicas part of the view if (i >= configurationStore.ReadViewHeadIndex) { this.lastRenewedReadView.Chain.Add(new Tuple <ReplicaInfo, CloudTableClient>(replica, tableClient)); } //Note the time when the view was updated this.lastViewRefreshTime = refreshStartTime; this.ConvertXStoreTableMode = configurationStore.ConvertXStoreTableMode; } } this.lastRenewedWriteView.ReadHeadIndex = configurationStore.ReadViewHeadIndex; break; } } } }