/// <summary>
/// Uploads the configuration to the cloud
/// </summary>
/// <returns></returns>
internal void UploadConfiguration()
{
try
{
using (MemoryStream stream = new MemoryStream())
{
BinaryFormatter formatter = new BinaryFormatter();
formatter.Serialize(stream, this.config);
stream.Position = 0;
AccessCondition access = (reconfigurationETag != null) ? AccessCondition.GenerateIfMatchCondition(reconfigurationETag) : null;
configurationBlob.UploadFromStream(stream, access);
}
}
catch (StorageException ex)
{
reconfigurationETag = null;
if (StorageExceptionCode.PreconditionFailed(ex))
{
// ETag condition was not met; there must be a concurrent reconfiguration and we lost
// so do nothing
Console.WriteLine("Reconfiguration aborted due to concurrent activity");
}
else
{
throw ex;
}
}
}
private void AcquireBlobLease(LeaseTakingPolicy policy)
{
string ProposedLeaseId = Guid.NewGuid().ToString();
bool isDone = false;
// if policy == LeaseTakingPolicy.TryUntilSuccessful then loop indefinitely
while (!isDone)
{
try
{
LeaseId = blob.AcquireLease(null, ProposedLeaseId);
HasLease = true;
isDone = true;
}
catch (StorageException ex)
{
HasLease = false;
if (StorageExceptionCode.Conflict(ex)) //Conflict
{
if (policy == LeaseTakingPolicy.TryOnce)
{
isDone = true;
}
}
else
{
ReleaseBlobLease();
throw;
}
}
}
}
private void ReleaseBlobLease()
{
try
{
if (LeaseId == null || LeaseId.Equals(""))
{
return;
}
blob.ReleaseLease(AccessCondition.GenerateLeaseCondition(LeaseId));
}
catch (StorageException ex)
{
if (StorageExceptionCode.NotFound(ex))
{
// Container is removed, hence its lease.
return;
}
else
{
throw ex;
}
}
catch (Exception exx)
{
throw exx;
}
}
/// <summary>
/// Renews a non-exclusive lease on the configuration.
/// If this returns a positive epoch number, then the lease as been acquired.
/// If a negative number is returned, then the lease was not renewed because either
/// there is a reconfiguration in progress or a configuration blob was not found.
/// </summary>
/// <returns>The epoch of the current leased configuration.</returns>
public int RenewLease()
{
DateTime startRPC = DateTime.Now;
try
{
configurationBlob.FetchAttributes();
}
catch (StorageException ex)
{
//304 is azure's not modified exception.
if (StorageExceptionCode.NotModified(ex))
{
// the configuration is not modified since last renewal
}
//404 is Azure's not-found exception
if (StorageExceptionCode.NotFound(ex))
{
return(-1);
}
throw ex;
}
int currentEpoch = Convert.ToInt32(configurationBlob.Metadata[ConstPool.EPOCH_NUMBER]);
if (currentEpoch > 0)
{
// no reconfiguration in progress so renewal was successful
expirationTime = startRPC.AddMilliseconds(ConstPool.CACHED_CONFIGURATION_VALIDITY_DURATION);
if (currentEpoch > config.Epoch)
{
// configuration has changed
DownloadConfiguration();
Console.WriteLine("New configuration for Epoch " + config.Epoch + " is primaries: " + String.Join(", ", config.PrimaryServers.ToList()) + " secondaries:" + String.Join(", ", config.SecondaryServers.ToList()));
}
}
return(currentEpoch);
}
/// <summary>
/// Starts a new configuration epoch and uploads the local configuration.
/// </summary>
/// <param name="updateConfiguration">If true, the configuration is also uploaded to the azure storage.</param>
/// <returns></returns>
internal void StartNewEpoch(bool uploadConfiguration = true)
{
if (uploadConfiguration)
{
Console.WriteLine("Uploading new configuration to the cloud...");
UploadConfiguration();
reconfigurationETag = configurationBlob.Properties.ETag;
}
try
{
int newEpoch = ++config.Epoch;
configurationBlob.Metadata[ConstPool.EPOCH_NUMBER] = "" + newEpoch;
configurationBlob.Metadata[ConstPool.EPOCH_MODIFIED_TIME] = DateTimeOffset.Now.ToString();
AccessCondition access = (reconfigurationETag != null) ? AccessCondition.GenerateIfMatchCondition(reconfigurationETag) : null;
configurationBlob.SetMetadata(access);
reconfigurationETag = null;
Console.WriteLine("Epoch " + config.Epoch + " written to the azure and started.");
}
catch (StorageException ex)
{
reconfigurationETag = null;
if (StorageExceptionCode.PreconditionFailed(ex))
{
// ETag condition was not met; there must be a concurrent reconfiguration and we lost
// Although our proposed configuration was already uploaded, we need to do nothing since
// it will be overwritten by the winning configurator.
Console.WriteLine("Reconfiguration aborted due to concurrent activity");
}
else
{
throw ex;
}
}
}
/// <summary>
/// Perform read optimistically and then verify configuration.
/// </summary>
/// <param name="op">The read operation</param>
/// <param name="blob">The blob being read</param>
/// <param name="ss">The chosen server</param>
/// <returns>whether the read succeeded; if not, then it should be tried again.</returns>
private void SlowRead(ReadOp op)
{
// TODO: Deal with the case that we try to read from a replica that is no longer a replica and the read fails
// In this case, the read should be retried after refreshing the configuration.
ServerState ss = null;
try
{
bool isDone = false;
do // until we enter fast mode or succeed at reading in slow mode with the correct configuration
{
ss = slaEngine.FindServerToRead(blobName);
blobForRead = ClientRegistry.GetCloudBlob(ss.Name, containerName, blobName);
// TODO: this should really check if the reader wants strong consistency
// It could be that eventual consistency is fine but the SLA Engine just happened to choose a primary server
// In this case, we can do a fast mode read since we don't care if the chosen server is no longer a primary
if (configuration.IsInFastMode() || !configuration.PrimaryServers.Contains(ss.Name))
{
// it is fine to read from the selected secondary replica
// or from a primary replica if we have now entered fast mode
FastRead(op);
isDone = true;
}
else
{
// read from the selected replica that we believe to be a primary replica
watch.Start();
op(blobForRead);
watch.Stop();
// then see if the configuration has changed and the selected replica is no longer primary
configuration.SyncWithCloud(ClientRegistry.GetConfigurationAccount());
// TODO: check the epoch number on the configuration to make sure that we have not missed a configuration.
// i.e. we need to make sure that the server from which we read did not become a secondary during our read,
// and then back to primary when we checked.
// TODO: maybe we should check that the read delivered a non-zero utility.
// It is possible that the configuration was so stale that a much better server could have been chosen.
if (configuration.PrimaryServers.Contains(ss.Name))
{
//We have contacted the primary replica, hence we are good to go.
slaEngine.RecordObjectRead(blobForRead.Name, Timestamp(blobForRead), ss, watch.ElapsedMilliseconds);
isDone = true;
}
isDone = false; // not done
}
} while (!isDone);
}
// TODO: decide if we need to catch any exceptions here or just let then propagate through
catch (StorageException se)
{
if (StorageExceptionCode.NotFound(se))
{
//blob is not found.
//this is fine because the replica might have removed.
//it simply returns, so client need to re-execute if this happens.
//We can also re-execute the read here, but for debugging purposes, it's simpler for the client to re-execute.
//Of course in real system, we need to re-execute at this level.
return;
}
// storage exceptions are passed through to the caller
throw;
}
catch (Exception ex)
{
if (ex.Message.Contains("Object reference not set to an instance of an object"))
{
return;
}
throw ex;
}
}
private void Sync()
{
try
{
//we first initialize a new replica in the new server
if (!targetContainer.Exists())
{
targetContainer.Create();
}
}
catch (StorageException ex)
{
//409 is the conflict message.
if (StorageExceptionCode.Conflict(ex))
{
throw ex;
}
}
if (lastModified == null)
{
targetContainer.FetchAttributes();
if (targetContainer.Metadata.ContainsKey("lastsync"))
{
//if no lastmodified time is provided in the constructor, we still try to be fast.
//So, we check to see if by any chance the container previously has synchronized.
lastModified = DateTimeOffset.Parse(targetContainer.Metadata["lastsync"]);
}
}
DateTimeOffset startTime = DateTimeOffset.Now;
try
{
IEnumerable <IListBlobItem> sourceBlobList = null;
if (lastModified != null)
{
sourceBlobList = sourceContainer.ListBlobs().OfType <ICloudBlob>().Where(b => b.Properties.LastModified > lastModified);
}
else
{
//this is the slowest sync path. It needs to go through all blobs.
//it tries to avoid this path as much as possible.
sourceBlobList = sourceContainer.ListBlobs();
}
foreach (ICloudBlob sourceBlob in sourceBlobList)
{
ICloudBlob targetBlob = targetContainer.GetBlockBlobReference(sourceBlob.Name);
//Since its an inter-account transfer, we need access rights to do the transfer.
var sas = sourceContainer.GetSharedAccessSignature(new SharedAccessBlobPolicy()
{
SharedAccessStartTime = DateTime.UtcNow.AddMinutes(-5),
SharedAccessExpiryTime = DateTime.UtcNow.AddMinutes(30),
Permissions = SharedAccessBlobPermissions.Read,
});
AccessCondition ac = new AccessCondition();
Interlocked.Increment(ref concurrentSyncs);
var srcBlockBlobSasUri = string.Format("{0}{1}", sourceBlob.Uri, sas);
targetBlob.BeginStartCopyFromBlob(new Uri(srcBlockBlobSasUri), BlobCopyFinished, null);
}
while (concurrentSyncs > 0)
{
lock (mylock)
Monitor.Wait(mylock, 200);
}
targetContainer.Metadata["lastsync"] = startTime.ToString();
targetContainer.SetMetadata();
}
catch (StorageException se)
{
//if the blob/container does not exist, it means that it is removed by configurator.
//We safely return.
if (StorageExceptionCode.NotFound(se))
{
return;
}
throw se;
}
catch (Exception ex)
{
throw ex;
}
}
/// <summary>
/// Read the state of the container configuration from the Azure storage.
/// </summary>
internal void ReadConfiguration()
{
ICloudBlob blob;
try
{
try
{
blob = GetConfigurationContainer().GetBlockBlobReference(ConstPool.CURRENT_CONFIGURATION_BLOB_NAME);
blob.FetchAttributes();
}
catch (StorageException se)
{
//304 is azure's not modified exception.
//it means that the configuration is not modified since last read.
if (StorageExceptionCode.NotModified(se))
{
Console.WriteLine("ReadConfiguration returned without reading...");
return;
}
//404 is Azure's not-found exception
if (StorageExceptionCode.NotFound(se))
{
Console.WriteLine("ReadConfiguration did not find a configuration blob; it needs to be created...");
PrimaryServers = null;
return;
}
Console.WriteLine(se.ToString());
Console.WriteLine(se.StackTrace);
throw se;
}
using (MemoryStream stream = new MemoryStream())
{
blob.DownloadToStream(stream);
stream.Seek(0, 0);
BinaryFormatter formatter = new BinaryFormatter();
ContainerConfiguration tmp = (ContainerConfiguration)formatter.Deserialize(stream);
if (!blob.Metadata.ContainsKey(ConstPool.EPOCH_NUMBER))
{
Console.WriteLine("No Epoch in configuration metadata!"); // this should not happen
return; // stay with current configuration for now
}
int currentEpoch = Convert.ToInt32(blob.Metadata[ConstPool.EPOCH_NUMBER]);
if (currentEpoch > Epoch)
{
PrimaryServers = tmp.PrimaryServers;
PrimaryServers.Sort();
SecondaryServers = tmp.SecondaryServers ?? new List <string>();
MainPrimaryIndex = tmp.MainPrimaryIndex;
WriteOnlyPrimaryServers = tmp.WriteOnlyPrimaryServers ?? new List <string>();
SyncPeriod = tmp.SyncPeriod;
SyncPrimaryServersWithSessionState();
SyncSecondaryServersWithSessionState();
garbageCollectOldServersFromSessionState();
Epoch = currentEpoch;
if (blob.Metadata[ConstPool.EPOCH_MODIFIED_TIME] != null)
{
EpochModifiedTime = DateTimeOffset.Parse(blob.Metadata[ConstPool.EPOCH_MODIFIED_TIME]);
}
else
{
EpochModifiedTime = DateTimeOffset.MinValue;
}
}
}
}
catch (StorageException se)
{
Console.WriteLine(se.ToString());
Console.WriteLine(se.StackTrace);
throw se;
}
catch (Exception ex)
{
Console.WriteLine(ex.ToString());
Console.WriteLine(ex.StackTrace);
//throw ex;
}
}
