public static List <ReplicatedTableReadBlobResult> TryReadAllBlobs <T>(List <CloudBlockBlob> blobs, out List <T> values, out List <string> eTags, Func <string, T> ParseBlobFunc)
where T : class
{
int numberOfBlobs = blobs.Count;
T[] valuesArray = new T[numberOfBlobs];
string[] eTagsArray = new string[numberOfBlobs];
ReplicatedTableReadBlobResult[] resultArray = new ReplicatedTableReadBlobResult[numberOfBlobs];
// read from all the blobs in parallel
Parallel.For(0, numberOfBlobs, index =>
{
DateTime startTime = DateTime.UtcNow;
T currentValue;
string currentETag;
resultArray[index] = TryReadBlob(blobs[index], out currentValue, out currentETag, ParseBlobFunc);
valuesArray[index] = currentValue;
eTagsArray[index] = currentETag;
ReplicatedTableLogger.LogInformational("TryReadBlob #{0} took {1}", index, DateTime.UtcNow - startTime);
});
values = valuesArray.ToList();
eTags = eTagsArray.ToList();
return(resultArray.ToList());
}
public static async Task <ReplicatedTableReadBlobResult> TryReadBlobAsync <T>(CloudBlockBlob blob, Action <T, string> callback, Func <string, T> ParseBlobFunc, CancellationToken ct)
where T : class
{
try
{
BlobRequestOptions options = new BlobRequestOptions()
{
ServerTimeout = TimeSpan.FromSeconds(5),
MaximumExecutionTime = TimeSpan.FromSeconds(30)
};
string content = await blob.DownloadTextAsync(null, null, options, null, ct);
if (content == Constants.ConfigurationStoreUpdatingText)
{
return(new ReplicatedTableReadBlobResult(ReadBlobCode.UpdateInProgress, "Blob update in progress ..."));
}
// ParseBlobFunc != null
T configuration = ParseBlobFunc(content);
string eTag = blob.Properties.ETag;
// callback != null
callback(configuration, eTag);
return(new ReplicatedTableReadBlobResult(ReadBlobCode.Success, ""));
}
catch (StorageException e)
{
var msg = string.Format("Error reading blob: {0}. StorageException: {1}", blob.Uri, e.Message);
ReplicatedTableLogger.LogError(msg);
if (e.RequestInformation != null &&
e.RequestInformation.HttpStatusCode == (int)HttpStatusCode.NotFound)
{
return(new ReplicatedTableReadBlobResult(ReadBlobCode.NotFound, msg));
}
return(new ReplicatedTableReadBlobResult(ReadBlobCode.StorageException, msg));
}
catch (OperationCanceledException)
{
var msg = string.Format("TryReadBlobAsync cancelled ({0})", blob.Uri);
ReplicatedTableLogger.LogInformational(msg);
return(new ReplicatedTableReadBlobResult(ReadBlobCode.Exception, msg));
}
catch (Exception e)
{
var msg = string.Format("Error reading blob: {0}. Exception: {1}", blob.Uri, e.Message);
ReplicatedTableLogger.LogError(msg);
return(new ReplicatedTableReadBlobResult(ReadBlobCode.Exception, msg));
}
}
private bool DoesViewNeedRefresh()
{
lock (this)
{
if ((DateTime.UtcNow - this.lastViewRefreshTime) >
TimeSpan.FromSeconds(Constants.LeaseRenewalIntervalInSec))
{
ReplicatedTableLogger.LogInformational("Need to renew lease on the view/refresh the view");
return(true);
}
return(false);
}
}
public static ReplicatedTableQuorumReadResult TryReadBlobQuorumFast <T>(List <CloudBlockBlob> blobs, out T value, out List <string> eTags, Func <string, T> ParseBlobFunc)
where T : class
{
value = default(T);
eTags = null;
int numberOfBlobs = blobs.Count;
var valuesArray = new List <T>(new T[numberOfBlobs]);
var eTagsArray = new List <string>(new string[numberOfBlobs]);
var resultArray = new List <ReplicatedTableReadBlobResult>(new ReplicatedTableReadBlobResult[numberOfBlobs]);
var cancel = new CancellationTokenSource();
/*
* Read async all the blobs in parallel
*/
Parallel.For(0, numberOfBlobs, async(index) =>
{
valuesArray[index] = default(T);
eTagsArray[index] = null;
resultArray[index] = new ReplicatedTableReadBlobResult(ReadBlobCode.NullObject, "downloaded not started yet!");
DateTime startTime = DateTime.UtcNow;
resultArray[index] = await TryReadBlobAsync(
blobs[index],
(currentValue, currentETag) =>
{
valuesArray[index] = currentValue;
eTagsArray[index] = currentETag;
},
ParseBlobFunc,
cancel.Token);
if (resultArray[index].Code == ReadBlobCode.Success)
{
ReplicatedTableLogger.LogInformational("TryReadBlobAsync #{0} took {1}", index, DateTime.UtcNow - startTime);
}
});
/*
* Poll for "Quorum" progress ...
*/
ReplicatedTableQuorumReadResult majority;
int quorumIndex;
do
{
Thread.Sleep(Constants.QuorumPollingInMilliSeconds);
// "resultArray" may change OOB just after Evaluate Quorum step below.
// In such case, we may exit the loop because all blobs are retrieved, while "majority" has a stale value!
// Therefore, we have to determine, before Evaluate Quorum step, if we'll exit the loop or no.
// If Exist, then "majority" would be final.
// If No, then we'll loop to compute the latest, unless we break because we already have Quorum without all blobs.
bool allBlobsRetrieved = true;
// IMPORTANT: foreach()/LINQ throws when "resultArray" changes (race condition).
for (int result = 0; result < resultArray.Count; result++)
{
if (resultArray[result].Code == ReadBlobCode.NullObject)
{
allBlobsRetrieved = false;
break;
}
}
// Evaluate Quorum ...
majority = FindMajority(resultArray.AsReadOnly(), valuesArray.AsReadOnly(), out quorumIndex);
if (majority.Code == ReplicatedTableQuorumReadCode.NotFound ||
majority.Code == ReplicatedTableQuorumReadCode.UpdateInProgress ||
majority.Code == ReplicatedTableQuorumReadCode.Exception ||
majority.Code == ReplicatedTableQuorumReadCode.Success)
{
// Quorum => cancel tasks and exit
cancel.Cancel();
break;
}
//else
if (allBlobsRetrieved)
{
// All blobs 'were' retrieved => exit
break;
}
// keep polling ...
} while (true);
cancel = null;
if (majority.Code == ReplicatedTableQuorumReadCode.Success)
{
value = valuesArray[quorumIndex];
eTags = eTagsArray;
}
return(majority);
}
private ReplicatedTableRepairResult RepairTable(string tableName, string storageAccountName, ReplicatedTableConfiguration configuration)
{
string viewName = "";
try
{
ReplicatedTableConfiguredTable tableConfig;
if (!configuration.IsConfiguredTable(tableName, out tableConfig))
{
return(new ReplicatedTableRepairResult(ReplicatedTableRepairCode.NotConfiguredTable, tableName));
}
viewName = tableConfig.ViewName;
List <ReplicaInfo> list = configuration.GetView(viewName).ReplicaChain;
if (!list.Any() ||
list[0].StorageAccountName != storageAccountName)
{
return(new ReplicatedTableRepairResult(ReplicatedTableRepairCode.NotImpactedTable, tableName, viewName, storageAccountName));
}
ReplicaInfo head = list[0];
if (head.Status != ReplicaStatus.WriteOnly)
{
return(new ReplicatedTableRepairResult(ReplicatedTableRepairCode.StableTable, tableName, viewName, storageAccountName));
}
int viewIdToRecoverFrom = (int)head.ViewWhenTurnedOff;
ReplicatedTableLogger.LogInformational("RepairTable={0}, View={1}, StorageAccountName={2}, from viewId={3} ...",
tableName,
viewName,
storageAccountName,
viewIdToRecoverFrom);
// Repairing ...
ReconfigurationStatus status = new ReplicatedTable(tableName, this).RepairTable(viewIdToRecoverFrom, null);
ReplicatedTableLogger.LogInformational("RepairTable={0}, View={1}, StorageAccountName={2}, from viewId={3} => Status={4}",
tableName,
viewName,
storageAccountName,
viewIdToRecoverFrom,
status);
if (status == ReconfigurationStatus.SUCCESS)
{
return(new ReplicatedTableRepairResult(ReplicatedTableRepairCode.Success, tableName, viewName, storageAccountName));
}
// Failure!
return(new ReplicatedTableRepairResult(ReplicatedTableRepairCode.Error, tableName, viewName, storageAccountName)
{
Status = status,
});
}
catch (Exception ex)
{
return(new ReplicatedTableRepairResult(ReplicatedTableRepairCode.Error, tableName, viewName, storageAccountName)
{
Status = ReconfigurationStatus.FAILURE,
Message = ex.ToString(),
});
}
}
public void TurnReplicaOn(string storageAccountName, List <string> tablesToRepair, out List <ReplicatedTableRepairResult> failures)
{
if (string.IsNullOrEmpty(storageAccountName))
{
throw new ArgumentNullException("storageAccountName");
}
if (tablesToRepair == null)
{
throw new ArgumentNullException("tablesToRepair");
}
ReplicatedTableConfiguration configuration = null;
failures = new List <ReplicatedTableRepairResult>();
// - Retrieve configuration ...
ReplicatedTableQuorumReadResult readResult = RetrieveConfiguration(out configuration);
if (readResult.Code != ReplicatedTableQuorumReadCode.Success)
{
var msg = string.Format("TurnReplicaOn={0}: failed to read configuration, \n{1}", storageAccountName, readResult.ToString());
ReplicatedTableLogger.LogError(msg);
throw new Exception(msg);
}
/* - Phase 1:
* Move the *replica* to the front and set it to None.
* Make the view ReadOnly.
**/
#region Phase 1
configuration.MoveReplicaToHeadAndSetViewToReadOnly(storageAccountName);
// - Write back configuration, refresh its Id with the new one,
// but don't validate it is loaded bcz if all views of the config are empty, the config won't be refreshed by RefreshReadAndWriteViewsFromBlobs() thread!
SaveConfigAndRefreshItsIdAndValidateIsLoaded(configuration, "Phase 1", false);
#endregion
/**
* Chain is such: [None] -> [RO] -> ... -> [RO]
* or: [None] -> [None] -> ... -> [None]
**/
// - Wait for L + CF to make sure no pending transaction working on old views
Thread.Sleep(TimeSpan.FromSeconds(Constants.LeaseDurationInSec + Constants.ClockFactorInSec));
/* - Phase 2:
* Set the *replica* (head) to WriteOnly and other active replicas to ReadWrite.
* Or, in case of one replic chain
* Set only the *replica* (head) to ReadWrite.
**/
#region Phase 2
configuration.EnableWriteOnReplicas(storageAccountName);
// - Write back configuration, refresh its Id with the new one,
// and then validate it is loaded now (it has to be working since next Phase is "Repair")
SaveConfigAndRefreshItsIdAndValidateIsLoaded(configuration, "Phase 2");
#endregion
/**
* Chain is such: [W] -> [RW] -> ... -> [RW]
* or: [RW] -> [None] -> ... -> [None]
**/
// To be safe:
// - Wait for L + CF to make sure no pending transaction working on old views
Thread.Sleep(TimeSpan.FromSeconds(Constants.LeaseDurationInSec + Constants.ClockFactorInSec));
/* - Phase 3:
* Repair all tables
**/
#region Phase 3
foreach (var tableName in tablesToRepair)
{
ReplicatedTableRepairResult result = RepairTable(tableName, storageAccountName, configuration);
if (result.Code != ReplicatedTableRepairCode.Error)
{
ReplicatedTableLogger.LogInformational(result.ToString());
continue;
}
ReplicatedTableLogger.LogError(result.ToString());
// List of tables (and corresponding views) failed to repair!
failures.Add(result);
}
#endregion
/* - Phase 4:
* Set the *replica* (head) to ReadWrite.
**/
#region Phase 4
configuration.EnableReadWriteOnReplicas(storageAccountName, failures.Select(r => r.ViewName).ToList());
// - Write back configuration, refresh its Id with the new one,
// and then validate it is loaded now (i.e. it is a working config)
SaveConfigAndRefreshItsIdAndValidateIsLoaded(configuration, "Phase 4");
#endregion
/**
* Chain is such: [RW] -> [RW] -> ... -> [RW] if all configured table repaired
* or: [W] -> [RW] -> ... -> [RW] if at least one configured table failed repair !
**/
}
private void RefreshReadAndWriteViewsFromBlobs(object arg)
{
List <ReplicatedTableConfiguredTable> tableConfigList;
int leaseDuration;
Guid configId;
List <View> views;
bool instrumentationFlag;
bool ignoreHigherViewIdRows;
// Lock because both SetConnectionStringStrategy and connectionStringMap can be updated OOB!
lock (connectionStringLock)
{
DateTime startTime = DateTime.UtcNow;
views = this.blobParser.ParseBlob(
this.blobs.Values.ToList(),
this.SetConnectionStringStrategy,
out tableConfigList,
out leaseDuration,
out configId,
out instrumentationFlag,
out ignoreHigherViewIdRows);
ReplicatedTableLogger.LogInformational("ParseBlob took {0}", DateTime.UtcNow - startTime);
if (views == null)
{
return;
}
}
lock (this)
{
// - Update lease duration
LeaseDuration = TimeSpan.FromSeconds(leaseDuration);
// - Update list of views
this.viewMap.Clear();
foreach (var view in views)
{
if (view == null || string.IsNullOrEmpty(view.Name))
{
continue;
}
// Set view LeaseDuration to the config LeaseDuration
view.LeaseDuration = LeaseDuration;
this.viewMap.Add(view.Name, view);
}
// - Update list of configured tables
this.tableMap.Clear();
defaultConfiguredRule = null;
if (tableConfigList != null)
{
foreach (var tableConfig in tableConfigList)
{
if (tableConfig == null || string.IsNullOrEmpty(tableConfig.TableName))
{
continue;
}
this.tableMap.Add(tableConfig.TableName, tableConfig);
if (tableConfig.UseAsDefault)
{
defaultConfiguredRule = tableConfig;
}
}
}
// - Update current config Id
currentRunningConfigId = configId;
// update instrumentation flag
this.instrumentation = instrumentationFlag;
// update ignoreHigherViewIdRows flag
this.ignoreHigherViewIdRows = ignoreHigherViewIdRows;
UpdateTimer();
}
}
private void RefreshReadAndWriteViewsFromBlobs(object arg)
{
lock (this)
{
this.lastRenewedReadView = new View();
this.lastRenewedWriteView = new View();
DateTime refreshStartTime = DateTime.UtcNow;
Dictionary <long, List <CloudBlockBlob> > viewResult = new Dictionary <long, List <CloudBlockBlob> >();
foreach (var blob in this.blobs)
{
ReplicatedTableConfigurationStore configurationStore;
if (!CloudBlobHelpers.TryReadBlob <ReplicatedTableConfigurationStore>(blob.Value, out configurationStore))
{
continue;
}
if (configurationStore.ViewId <= 0)
{
ReplicatedTableLogger.LogInformational("ViewId={0} is invalid. Must be >= 1. Skipping this blob {1}.",
configurationStore.ViewId,
blob.Value.Uri);
continue;
}
List <CloudBlockBlob> viewBlobs;
if (!viewResult.TryGetValue(configurationStore.ViewId, out viewBlobs))
{
viewBlobs = new List <CloudBlockBlob>();
viewResult.Add(configurationStore.ViewId, viewBlobs);
}
viewBlobs.Add(blob.Value);
if (viewBlobs.Count >= this.quorumSize)
{
this.lastRenewedReadView.ViewId = this.lastRenewedWriteView.ViewId = configurationStore.ViewId;
for (int i = 0; i < configurationStore.ReplicaChain.Count; i++)
{
ReplicaInfo replica = configurationStore.ReplicaChain[i];
CloudTableClient tableClient = GetTableClientForReplica(replica);
if (replica != null && tableClient != null)
{
//Update the write view always
this.lastRenewedWriteView.Chain.Add(new Tuple <ReplicaInfo, CloudTableClient>(replica, tableClient));
//Update the read view only for replicas part of the view
if (i >= configurationStore.ReadViewHeadIndex)
{
this.lastRenewedReadView.Chain.Add(new Tuple <ReplicaInfo, CloudTableClient>(replica, tableClient));
}
//Note the time when the view was updated
this.lastViewRefreshTime = refreshStartTime;
this.ConvertXStoreTableMode = configurationStore.ConvertXStoreTableMode;
}
}
this.lastRenewedWriteView.ReadHeadIndex = configurationStore.ReadViewHeadIndex;
break;
}
}
}
}
