public LocationConstraint(string containerName, ReplicaConfiguration currentConfig, string serverName, LocationConstraintType type)
: base(containerName, currentConfig)
{
this.Type = type;
this.ConstrainedContainer = ClientRegistry.GetCloudBlobContainer(serverName, containerName);
this.ServerName = serverName;
}
private void FastWrite(WriteOp op, List <SessionState> sessions = null, ServerMonitor monitor = null)
{
// there should only be one primary server since we are in fast mode
string server = configuration.PrimaryServers.First();
ICloudBlob blob = ClientRegistry.GetCloudBlob(server, configuration.Name, blobName, false);
watch.Start();
op(blob);
watch.Stop();
// update server and session state
ServerState ss = (monitor == null) ? slaEngine.Monitor.GetServerState(server) : monitor.GetServerState(server);
ss.AddRtt(watch.ElapsedMilliseconds);
if (sessions == null)
{
slaEngine.Session.RecordObjectWritten(blobName, Timestamp(blob), ss);
}
else
{
foreach (SessionState session in sessions)
{
session.RecordObjectWritten(blobName, Timestamp(blob), ss);
}
}
}
public override void Execute()
{
CloudBlobContainer primaryContainer = ClientRegistry.GetCloudBlobContainer(Configuration.PrimaryServers.First(), ModifyingContainer.Name);
SynchronizeContainer synchronizer = new SynchronizeContainer(primaryContainer, ModifyingContainer);
synchronizer.BeginSyncContainers();
Configuration.EndCurrentEpoch();
Configuration.PrimaryServers.Add(ServerName);
//we add the new primary also to the list of write_only primaries.
//Servers in this list are not registered in session state, hence a get operation cannot be issued for them.
//This makes put operations to go to all primaries (including the new one), but get_primary only goes to old primaries (those that does not exist in not register list).
Configuration.WriteOnlyPrimaryServers.Add(ServerName);
Configuration.StartNewEpoch();
//We wait until sync is finihsed
synchronizer.EndSyncContainers();
// TODO: check that the new primary server did indeed get all of the writes.
Configuration.EndCurrentEpoch();
//We clear the not register primary list, so the new primary can also be registered and used for get_primary operations.
Configuration.WriteOnlyPrimaryServers.Clear();
//Finally, we update the lookup service by writing the new configuration to the cloud, so every other client can read it.
Configuration.StartNewEpoch();
}
/// <summary>
/// Sets the WiP flags on all of the primary blobs.
/// </summary>
/// <returns>a dictionary containing the returned eTags for each primary;
/// returns null if the flag-setting protocol was not successful</returns>
private Dictionary <string, string> SetWiPFlags()
{
ICloudBlob blob;
Dictionary <string, string> eTags = new Dictionary <string, string>();
bool didAtLeastOne = false;
foreach (string server in configuration.PrimaryServers)
{
blob = ClientRegistry.GetCloudBlob(server, configuration.Name, blobName, false);
blob.Metadata[ConstPool.WRITE_IN_PROGRESS] = ConstPool.WRITE_IN_PROGRESS;
try
{
blob.SetMetadata();
didAtLeastOne = true;
}
catch (StorageException)
{
// If setting the flag fails at some primary, then abort the protocol
// Note that some WiP flags may have already been set, so we first try to clear them
if (didAtLeastOne)
{
ClearWiPFlags();
}
return(null);
}
eTags[server] = blob.Properties.ETag;
}
return(eTags);
}
private void DoDelete(Microsoft.WindowsAzure.Storage.Blob.DeleteSnapshotsOption deleteSnapshotsOption = DeleteSnapshotsOption.None, Microsoft.WindowsAzure.Storage.AccessCondition accessCondition = null, Microsoft.WindowsAzure.Storage.Blob.BlobRequestOptions options = null, Microsoft.WindowsAzure.Storage.OperationContext operationContext = null)
{
bool done = false;
while (!done)
{
using (PrimaryCloudBlobLease lease = new PrimaryCloudBlobLease(this.Name, configuration, true))
{
if (lease.HasLease)
{
Dictionary <ICloudBlob, IAsyncResult> results = new Dictionary <ICloudBlob, IAsyncResult>();
foreach (string serverName in configuration.PrimaryServers)
{
watch.Start();
ICloudBlob blob = ClientRegistry.GetCloudBlob(serverName, configuration.Name, Name);
results[blob] = blob.BeginDelete(deleteSnapshotsOption, lease.getAccessConditionWithLeaseId(accessCondition), options, operationContext, null, null);
ServerState ss = slaEngine.Monitor.GetServerState(serverName);
ss.AddRtt(watch.ElapsedMilliseconds);
slaEngine.Session.RecordObjectWritten(Name, Timestamp(blob), ss);
}
foreach (ICloudBlob blob in results.Keys)
{
blob.EndDelete(results[blob]);
}
done = true;
}
}
}
}
internal ConfigurationAction(string containerName, string serverName, float?utility, string SLAId = null, ConfigurationActionSource source = ConfigurationActionSource.NonConstraint, int numberOfReads = 0, int numberOfWrites = 0)
{
this.Configuration = ClientRegistry.GetConfiguration(containerName, false);
this.ModifyingContainer = ClientRegistry.GetCloudBlobContainer(serverName, containerName);
this.GainedUtility = utility ?? 0;
OriginatingSLAs = new HashSet <string>();
Clients = new HashSet <string>();
if (SLAId != null)
{
OriginatingSLAs.Add(SLAId);
}
this.Source = source;
this.numberOfReads = numberOfReads;
this.numberOfWrites = numberOfWrites;
this.ServerName = serverName;
if (numberOfBlobs == 0)
{
CloudBlobContainer primaryContainer = ClientRegistry.GetCloudBlobContainer(Configuration.PrimaryServers.First(), containerName);
numberOfBlobs = primaryContainer.ListBlobs().Count();
}
Cost = ComputeCost();
}
public static void Initialize()
{
// get list of Azure storage accounts
accounts = PileusApp.Account.GetStorageAccounts(false);
ClientRegistry.Init(accounts, accounts[configStorageSite]);
// delete cloud configuration if desired
if (clearConfiguration)
{
ReplicaConfiguration configToDelete = new ReplicaConfiguration(containerName);
ConfigurationCloudStore backingStore = new ConfigurationCloudStore(accounts[configStorageSite], configToDelete);
Log("Deleting configuration in Azure...");
backingStore.DeleteConfiguration();
Log("Done. Now waiting for it to really take effect...");
Thread.Sleep(40000); // give system a chance to complete delete
}
// read/create configuration
Log("Creating/reading configuration...");
config = new ReplicaConfiguration(containerName, PrimaryServers, SecondaryServers, NonReplicaServers, ReadOnlySecondaryServers, cloudBackedConfiguration, stableConfiguration);
ClientRegistry.AddConfiguration(config);
// upload data into containers if desired
if (reloadDatabase)
{
CreateDatabase(numBlobs);
}
// create SLAs
slas = new Dictionary <string, ServiceLevelAgreement>();
slas["strong"] = CreateConsistencySla(Consistency.Strong);
slas["causal"] = CreateConsistencySla(Consistency.Causal);
slas["bounded"] = CreateBoundedConsistencySla(boundInSeconds);
slas["readmywrites"] = CreateConsistencySla(Consistency.ReadMyWrites);
slas["monotonic"] = CreateConsistencySla(Consistency.MonotonicReads);
slas["eventual"] = CreateConsistencySla(Consistency.Eventual);
currentSLA = CreateFastOrStrongSla();
slas["sla"] = currentSLA;
// get/create replicated container
Log("Creating replicated container...");
Dictionary <string, CloudBlobContainer> containers = new Dictionary <string, CloudBlobContainer>();
foreach (string site in accounts.Keys)
{
CloudBlobClient blobClient = accounts[site].CreateCloudBlobClient();
CloudBlobContainer blobContainer = blobClient.GetContainerReference(containerName);
blobContainer.CreateIfNotExists();
containers.Add(site, blobContainer);
}
container = new CapCloudBlobContainer(containers, PrimaryServers.First());
container.Configuration = config;
container.Monitor = new ServerMonitor(config);
configurator = new Configurator(containerName);
}
public override void Execute()
{
if (!Configuration.PrimaryServers.Contains(ServerName))
{
AppendToLogger("Start Asynchronous Synchronization.");
CloudBlobContainer primaryContainer = ClientRegistry.GetCloudBlobContainer(Configuration.PrimaryServers.First(), ModifyingContainer.Name);
SynchronizeContainer synchronizer = new SynchronizeContainer(primaryContainer, ModifyingContainer);
// let's try avoiding asynchronous sync to see if that makes things run faster
// synchronizer.BeginSyncContainers();
AppendToLogger("Ending Epoch " + Configuration.Epoch);
Configuration.EndCurrentEpoch();
Configuration.PrimaryServers.Add(ServerName);
//add the new primary also to the list of write_only primaries.
//Servers in this list are not registered in session state, hence a get operation cannot be issued for them.
//This makes put operations to go to all primaries (including the new one), but get_primary only goes to old primaries (those that does not exist in not register list).
Configuration.WriteOnlyPrimaryServers.Add(ServerName);
AppendToLogger("Starting the new Epoch");
Configuration.StartNewEpoch();
//wait until sync is finihsed
AppendToLogger("Wait to finish Synchronization");
// synchronizer.EndSyncContainers();
synchronizer.SyncContainers();
}
AppendToLogger("Synchronization finished. Ending current epoch again.");
Configuration.EndCurrentEpoch();
//clear the not registered primary list, so the new primary can also be registered and used for get_primary operations.
Configuration.WriteOnlyPrimaryServers.Clear();
//all primaries will become secondaries, hence they are added to secondary list, and their sync period is set.
Configuration.PrimaryServers.ForEach(s => { if (!s.Equals(ServerName))
{
Configuration.SecondaryServers.Add(s);
}
Configuration.SetSyncPeriod(s, ConstPool.DEFAULT_SYNC_INTERVAL); });
Configuration.PrimaryServers.Clear();
//new primary is added to primary list.
Configuration.PrimaryServers.Add(ServerName);
//if the new primary has been a secondary, it is removed from the secondary list.
if (Configuration.SecondaryServers.Contains(ServerName))
{
Configuration.SecondaryServers.Remove(ServerName);
}
//Finally, update the lookup service by writing the new configuration to the cloud, so every other client can read it.
AppendToLogger("Starting the new Epoch with One Primary");
Configuration.StartNewEpoch();
}
/// <summary>
/// Pings all servers to obtain their round-trips times and their high timestamps.
/// </summary>
public void PingTimestampsNow()
{
Stopwatch watch = new Stopwatch();
foreach (string server in configuration.SecondaryServers)
{
// if the server is not reached yet, we perform a dummy operation for it.
if (!replicas[server].IsContacted())
{
try
{
//we perform a dummy operation to get the rtt latency!
DateTimeOffset?serverTime = null;
long rtt;
if (server.EndsWith("-secondary") && configuration.PrimaryServers.Contains(server.Replace("-secondary", "")))
{
// get the server's last sync time from Azure
// this call only works on Azure secondaries
CloudBlobClient blobClient = ClientRegistry.GetCloudBlobClient(server);
watch.Restart();
ServiceStats stats = blobClient.GetServiceStats();
rtt = watch.ElapsedMilliseconds;
replicas[server].AddRtt(rtt);
if (stats.GeoReplication.LastSyncTime.HasValue)
{
serverTime = stats.GeoReplication.LastSyncTime.Value;
}
}
else
{
// get the server's last sync time from the container's metadata
CloudBlobContainer blobContainer = ClientRegistry.GetCloudBlobContainer(server, configuration.Name);
watch.Restart();
blobContainer.FetchAttributes();
rtt = watch.ElapsedMilliseconds;
if (blobContainer.Metadata.ContainsKey("lastsync"))
{
//if no lastmodified time is provided in the constructor, we still try to be fast.
//So, we check to see if by any chance the container previously has synchronized.
serverTime = DateTimeOffset.Parse(blobContainer.Metadata["lastsync"]);
}
}
if (serverTime.HasValue && serverTime > replicas[server].HighTime)
{
replicas[server].HighTime = serverTime.Value;
}
}
catch (StorageException)
{
// do nothing
}
}
}
}
/// <summary>
/// Returns the blob for the main primary site of the current configuration.
/// </summary>
/// <returns>the primary blob (or null if unable to renew the lease on the configuration)</returns>
public ICloudBlob MainPrimary()
{
ICloudBlob primary = null;
if (configuration.IsInFastMode(renew: true))
{
string server = configuration.PrimaryServers.First();
primary = ClientRegistry.GetCloudBlob(server, configuration.Name, blobName);
}
return(primary);
}
public override double ComputeCost()
{
double result = 0;
double transactionalCost = CostModel.GetSecondaryTransactionalCost(numberOfReads);
double syncCost = CostModel.GetSyncCost(numberOfWrites, ConstPool.DEFAULT_SYNC_INTERVAL);
double StorageCost = CostModel.GetStorageCost(ClientRegistry.GetMainPrimaryContainer(Configuration.Name));
result = transactionalCost + syncCost + StorageCost;
return(result);
}
public void FetchAttributes(Microsoft.WindowsAzure.Storage.AccessCondition accessCondition = null, Microsoft.WindowsAzure.Storage.Blob.BlobRequestOptions options = null, Microsoft.WindowsAzure.Storage.OperationContext operationContext = null)
{
// TODO: use protocol
ServerState ss = slaEngine.FindServerToRead(Name);
ICloudBlob blob = ClientRegistry.GetCloudBlob(ss.Name, configuration.Name, Name);
watch.Start();
blob.FetchAttributes(accessCondition, options, operationContext);
ss.AddRtt(watch.ElapsedMilliseconds);
// slaEngine.SessionState.RecordObjectRead(blob.Name, Timestamp(blob), ss);
slaEngine.Session.RecordObjectRead(blob.Name, Timestamp(blob), ss, "");
}
/// <summary>
/// Perform standard read and update server and session state.
/// </summary>
/// <param name="op">The read operation</param>
/// <param name="blob">The blob being read</param>
/// <param name="ss">The chosen server</param>
private void FastRead(ReadOp op)
{
ServerState ss = slaEngine.FindServerToRead(blobName);
blobForRead = ClientRegistry.GetCloudBlob(ss.Name, containerName, blobName);
watch.Start();
op(blobForRead);
watch.Stop();
slaEngine.RecordObjectRead(blobForRead.Name, Timestamp(blobForRead), ss, watch.ElapsedMilliseconds);
}
/// <summary>
/// Perform a local write operation, e.g. set a property, on all blobs holding a replica.
/// This does not bother to check the lease on the configuration.
/// It should not be used for arbitrary writes that need to be performed atomically.
/// </summary>
/// <param name="op">the write operation being done</param>
public void SetProperty(WriteOp op)
{
foreach (string server in configuration.PrimaryServers)
{
ICloudBlob blob = ClientRegistry.GetCloudBlob(server, configuration.Name, blobName);
op(blob);
}
foreach (string server in configuration.SecondaryServers)
{
ICloudBlob blob = ClientRegistry.GetCloudBlob(server, configuration.Name, blobName);
op(blob);
}
}
public override void Execute()
{
AppendToLogger("Start Synchronization");
CloudBlobContainer primaryContainer = ClientRegistry.GetCloudBlobContainer(Configuration.PrimaryServers.First(), ModifyingContainer.Name);
SynchronizeContainer synchronizer = new SynchronizeContainer(primaryContainer, ModifyingContainer);
synchronizer.SyncContainers();
AppendToLogger("Synchronization finished.");
//Update the configuration
Configuration.SecondaryServers.Add(ServerName);
AppendToLogger("Starting the new Epoch");
Configuration.StartNewEpoch();
}
/// <summary>
/// Lease configuration container responsible for this blob along with blobs in the containers
/// </summary>
/// <param name="blobName">Particular blobs we would like to lease</param>
/// <param name="containerElementSet">list of containers</param>
/// <param name="putOptimization">perform put optimization. I.e., for put operations, it is not required to take lease if there is only one primary.</param>
public PrimaryCloudBlobLease(string blobName, ReplicaConfiguration configuration, bool putOptimization = false)
{
if ((putOptimization) && configuration.PrimaryServers.Count == 1)
{
//There is only one primary. No need to
HasLease = true;
return;
}
leasedBlobs = new Dictionary <ICloudBlob, string>();
//Reconfiguration is not going to happen in near future.
//We can safely take leases of primary blobs.
this.ProposedLeaseId = Guid.NewGuid().ToString();
foreach (string serverName in configuration.PrimaryServers)
{
try
{
ICloudBlob blob = ClientRegistry.GetCloudBlob(serverName, configuration.Name, blobName);
if (!blob.Exists())
{
//we cannot take lease on a non-existing blob.
//Hence, we create it first.
byte[] dummy = new byte[1];
dummy[0] = (byte)0;
var ms = new MemoryStream(dummy);
blob.UploadFromStream(ms, null, null, null);
}
string leaseID = blob.AcquireLease(null, ProposedLeaseId);
leasedBlobs.Add(blob, leaseID);
}
catch (StorageException ex)
{
releaseContainers();
leasedBlobs.Clear();
HasLease = false;
if (ex.GetBaseException().Message.Contains("409")) //Conflict
{
return;
}
else
{
throw ex;
}
}
}
HasLease = true;
}
private static void SyncSecondaryServers()
{
string primarySite = config.PrimaryServers.First();
foreach (string secondarySite in config.SecondaryServers)
{
if (!config.ReadOnlySecondaryServers.Contains(secondarySite))
{
SynchronizeContainer synchronizer = new SynchronizeContainer(ClientRegistry.GetCloudBlobContainer(primarySite, containerName),
ClientRegistry.GetCloudBlobContainer(secondarySite, containerName));
synchronizer.SyncContainers();
Log("Synchronization completed to " + SiteName(secondarySite) + ".");
}
else
{
Log("Could not sync to " + SiteName(secondarySite) + " since secondary is read-only.");
}
}
}
/// <summary>
/// Pings all servers to obtain their round-trips times and their high timestamps.
/// </summary>
public void PingNow()
{
Stopwatch watch = new Stopwatch();
foreach (string server in replicas.Keys)
{
// if the server is not reached yet, we perform a dummy operation for it.
if (!replicas[server].IsContacted())
{
//we perform a dummy operation to get the rtt latency!
CloudBlobClient blobClient = ClientRegistry.GetCloudBlobClient(server);
long rtt;
watch.Restart();
blobClient.GetServiceProperties();
rtt = watch.ElapsedMilliseconds;
replicas[server].AddRtt(rtt);
}
}
}
/// <summary>
/// Clears the WiP flags on all of the primary blobs.
/// </summary>
/// <param name="eTags">if not null, then the flags should be cleared conditionally</param>
private void ClearWiPFlags(Dictionary <string, string> eTags = null)
{
ICloudBlob blob;
AccessCondition access = AccessCondition.GenerateEmptyCondition();
// Clear WiP flags at non-main primaries first
foreach (string server in configuration.PrimaryServers.Skip(1))
{
blob = ClientRegistry.GetCloudBlob(server, configuration.Name, blobName, false);
blob.Metadata.Remove(ConstPool.WRITE_IN_PROGRESS);
if (eTags != null)
{
access = AccessCondition.GenerateIfMatchCondition(eTags[server]);
}
try
{
blob.SetMetadata(access);
}
catch (StorageException)
{
// Ignore failures since the only consequence is that the Wip flag remains set
// It could be that another write is still in progress
// The flag will be cleared eventually by another writer or by the recovery process
}
}
// Clear WiP on main primary last
blob = ClientRegistry.GetCloudBlob(configuration.PrimaryServers.First(), configuration.Name, blobName, false);
blob.Metadata.Remove(ConstPool.WRITE_IN_PROGRESS);
if (eTags != null)
{
access = AccessCondition.GenerateIfMatchCondition(eTags[configuration.PrimaryServers.First()]);
}
try
{
blob.SetMetadata(access);
}
catch (StorageException)
{
// Ignore
}
}
private void MultiWriteUsingBlobLease(WriteOp op)
{
// TODO: recover from failed clients that may leave a write partially completed
// TODO: remove use of leases and replace with ETags
// throw new Exception("Write to multiple primaries not yet implemented.");
try
{
bool done = false;
while (!done)
{
// grab lease on blob to guard against concurrent writers
using (PrimaryCloudBlobLease lease = new PrimaryCloudBlobLease(blobName, configuration, true))
{
if (lease.HasLease)
{
// TODO: fix code for writing to multiple primaries (and remove it from here)
foreach (string server in configuration.PrimaryServers)
{
watch.Start();
ICloudBlob blob = ClientRegistry.GetCloudBlob(server, configuration.Name, blobName, false);
op(blob);
watch.Stop();
ServerState ss = slaEngine.Monitor.GetServerState(server);
ss.AddRtt(watch.ElapsedMilliseconds);
slaEngine.Session.RecordObjectWritten(blobName, Timestamp(blob), ss);
}
done = true;
}
}
}
}
catch (StorageException se)
{
throw se;
}
catch (Exception ex)
{
throw ex;
}
}
public static string PrintCurrentConfiguration()
{
string result = null;
ReplicaConfiguration config = ClientRegistry.GetConfiguration(containerName, false);
result += "Current configuration for " + config.Name + ":" + "\r\n";
result += "Primary: ";
bool first = true;
foreach (string name in config.PrimaryServers)
{
if (first)
{
first = false;
}
else
{
result += ", ";
}
result += SiteName(name);
}
;
result += "\r\n";
result += "Secondaries: ";
first = true;
foreach (string name in config.SecondaryServers)
{
if (first)
{
first = false;
}
else
{
result += ", ";
}
result += SiteName(name);
}
;
result += "\r\n";
return(result);
}
private void DoUploadFromStream(System.IO.Stream source, Microsoft.WindowsAzure.Storage.AccessCondition accessCondition = null, Microsoft.WindowsAzure.Storage.Blob.BlobRequestOptions options = null, Microsoft.WindowsAzure.Storage.OperationContext operationContext = null)
{
try
{
bool done = false;
while (!done)
{
using (PrimaryCloudBlobLease lease = new PrimaryCloudBlobLease(Name, configuration, true))
{
if (lease.HasLease)
{
foreach (string server in configuration.PrimaryServers)
{
watch.Start();
ICloudBlob blob = ClientRegistry.GetCloudBlob(server, configuration.Name, Name, false);
source.Position = 0;
blob.UploadFromStream(source, lease.getAccessConditionWithLeaseId(accessCondition), options, operationContext);
watch.Stop();
ServerState ss = slaEngine.Monitor.GetServerState(server);
ss.AddRtt(watch.ElapsedMilliseconds);
slaEngine.Session.RecordObjectWritten(Name, Timestamp(blob), ss);
}
done = true;
}
}
}
}
catch (StorageException se)
{
throw se;
}
catch (Exception ex)
{
throw ex;
}
}
//public Microsoft.WindowsAzure.Storage.ICancellableAsyncResult BeginUploadFromStream(System.IO.Stream source, AsyncCallback callback, object state)
//{
// return strongBlob.BeginUploadFromStream(source, callback, state);
//}
//public Microsoft.WindowsAzure.Storage.ICancellableAsyncResult BeginUploadFromStream(System.IO.Stream source, Microsoft.WindowsAzure.Storage.AccessCondition accessCondition, Microsoft.WindowsAzure.Storage.Blob.BlobRequestOptions options, Microsoft.WindowsAzure.Storage.OperationContext operationContext, AsyncCallback callback, object state)
//{
// return strongBlob.BeginUploadFromStream(source, accessCondition, options, operationContext, callback, state);
//}
//public void EndUploadFromStream(IAsyncResult asyncResult)
//{
// strongBlob.EndUploadFromStream(asyncResult);
// slaEngine.SessionState.RecordObjectWritten(strongBlob.Name, Timestamp(strongBlob), primaryServer);
//}
//public Microsoft.WindowsAzure.Storage.ICancellableAsyncResult BeginDownloadToStream(System.IO.Stream target, AsyncCallback callback, object state)
//{
// // TODO: Use SLA to decide from which server to download.
// return strongBlob.BeginDownloadToStream(target, callback, state);
//}
//public Microsoft.WindowsAzure.Storage.ICancellableAsyncResult BeginDownloadToStream(System.IO.Stream target, Microsoft.WindowsAzure.Storage.AccessCondition accessCondition, Microsoft.WindowsAzure.Storage.Blob.BlobRequestOptions options, Microsoft.WindowsAzure.Storage.OperationContext operationContext, AsyncCallback callback, object state)
//{
// return strongBlob.BeginDownloadToStream(target, accessCondition, options, operationContext, callback, state);
//}
//public void EndDownloadToStream(IAsyncResult asyncResult)
//{
// strongBlob.EndDownloadToStream(asyncResult);
// slaEngine.SessionState.RecordObjectRead(strongBlob.Name, Timestamp(strongBlob), primaryServer);
//}
//public Microsoft.WindowsAzure.Storage.ICancellableAsyncResult BeginDownloadRangeToStream(System.IO.Stream target, long? offset, long? length, AsyncCallback callback, object state)
//{
// return strongBlob.BeginDownloadRangeToStream(target, offset, length, callback, state);
//}
//public Microsoft.WindowsAzure.Storage.ICancellableAsyncResult BeginDownloadRangeToStream(System.IO.Stream target, long? offset, long? length, Microsoft.WindowsAzure.Storage.AccessCondition accessCondition, Microsoft.WindowsAzure.Storage.Blob.BlobRequestOptions options, Microsoft.WindowsAzure.Storage.OperationContext operationContext, AsyncCallback callback, object state)
//{
// return strongBlob.BeginDownloadRangeToStream(target, offset, length, accessCondition, options, operationContext, callback, state);
//}
//public void EndDownloadRangeToStream(IAsyncResult asyncResult)
//{
// strongBlob.EndDownloadRangeToStream(asyncResult);
// slaEngine.SessionState.RecordObjectRead(strongBlob.Name, Timestamp(strongBlob), primaryServer);
//}
//public Microsoft.WindowsAzure.Storage.ICancellableAsyncResult BeginExists(AsyncCallback callback, object state)
//{
// return strongBlob.BeginExists(callback, state);
//}
//public Microsoft.WindowsAzure.Storage.ICancellableAsyncResult BeginExists(Microsoft.WindowsAzure.Storage.Blob.BlobRequestOptions options, Microsoft.WindowsAzure.Storage.OperationContext operationContext, AsyncCallback callback, object state)
//{
// return strongBlob.BeginExists(options, operationContext, callback, state);
//}
//public bool EndExists(IAsyncResult asyncResult)
//{
// bool result = strongBlob.EndExists(asyncResult);
// slaEngine.SessionState.RecordObjectRead(strongBlob.Name, Timestamp(strongBlob), primaryServer);
// return result;
//}
//public Microsoft.WindowsAzure.Storage.ICancellableAsyncResult BeginFetchAttributes(AsyncCallback callback, object state)
//{
// return strongBlob.BeginFetchAttributes(callback, state);
//}
//public Microsoft.WindowsAzure.Storage.ICancellableAsyncResult BeginFetchAttributes(Microsoft.WindowsAzure.Storage.AccessCondition accessCondition, Microsoft.WindowsAzure.Storage.Blob.BlobRequestOptions options, Microsoft.WindowsAzure.Storage.OperationContext operationContext, AsyncCallback callback, object state)
//{
// return strongBlob.BeginFetchAttributes(accessCondition, options, operationContext, callback, state);
//}
//public void EndFetchAttributes(IAsyncResult asyncResult)
//{
// strongBlob.EndFetchAttributes(asyncResult);
// slaEngine.SessionState.RecordObjectRead(strongBlob.Name, Timestamp(strongBlob), primaryServer);
//}
//public void SetMetadata(Microsoft.WindowsAzure.Storage.AccessCondition accessCondition = null, Microsoft.WindowsAzure.Storage.Blob.BlobRequestOptions options = null, Microsoft.WindowsAzure.Storage.OperationContext operationContext = null)
//{
// watch.Start();
// strongBlob.SetMetadata(accessCondition, options, operationContext);
// primaryServer.AddRtt(watch.ElapsedMilliseconds);
// slaEngine.SessionState.RecordObjectWritten(strongBlob.Name, Timestamp(strongBlob), primaryServer);
//}
//public Microsoft.WindowsAzure.Storage.ICancellableAsyncResult BeginSetMetadata(AsyncCallback callback, object state)
//{
// return strongBlob.BeginSetMetadata(callback, state);
//}
//public Microsoft.WindowsAzure.Storage.ICancellableAsyncResult BeginSetMetadata(Microsoft.WindowsAzure.Storage.AccessCondition accessCondition, Microsoft.WindowsAzure.Storage.Blob.BlobRequestOptions options, Microsoft.WindowsAzure.Storage.OperationContext operationContext, AsyncCallback callback, object state)
//{
// return strongBlob.BeginSetMetadata(accessCondition, options, operationContext, callback, state);
//}
//public void EndSetMetadata(IAsyncResult asyncResult)
//{
// strongBlob.EndSetMetadata(asyncResult);
// slaEngine.SessionState.RecordObjectWritten(strongBlob.Name, Timestamp(strongBlob), primaryServer);
//}
//public void SetProperties(Microsoft.WindowsAzure.Storage.AccessCondition accessCondition = null, Microsoft.WindowsAzure.Storage.Blob.BlobRequestOptions options = null, Microsoft.WindowsAzure.Storage.OperationContext operationContext = null)
//{
// watch.Start();
// strongBlob.SetProperties(accessCondition, options, operationContext);
// primaryServer.AddRtt(watch.ElapsedMilliseconds);
// slaEngine.SessionState.RecordObjectWritten(strongBlob.Name, Timestamp(strongBlob), primaryServer);
//}
//public Microsoft.WindowsAzure.Storage.ICancellableAsyncResult BeginSetProperties(AsyncCallback callback, object state)
//{
// return strongBlob.BeginSetProperties(callback, state);
//}
//public Microsoft.WindowsAzure.Storage.ICancellableAsyncResult BeginSetProperties(Microsoft.WindowsAzure.Storage.AccessCondition accessCondition, Microsoft.WindowsAzure.Storage.Blob.BlobRequestOptions options, Microsoft.WindowsAzure.Storage.OperationContext operationContext, AsyncCallback callback, object state)
//{
// return strongBlob.BeginSetProperties(accessCondition, options, operationContext, callback, state);
//}
//public void EndSetProperties(IAsyncResult asyncResult)
//{
// strongBlob.EndSetProperties(asyncResult);
// slaEngine.SessionState.RecordObjectWritten(strongBlob.Name, Timestamp(strongBlob), primaryServer);
//}
public void Delete(Microsoft.WindowsAzure.Storage.Blob.DeleteSnapshotsOption deleteSnapshotsOption = DeleteSnapshotsOption.None, Microsoft.WindowsAzure.Storage.AccessCondition accessCondition = null, Microsoft.WindowsAzure.Storage.Blob.BlobRequestOptions options = null, Microsoft.WindowsAzure.Storage.OperationContext operationContext = null)
{
bool isDone = false;
do
{
try
{
if (configuration.IsInFastMode())
{
DoDelete(deleteSnapshotsOption, accessCondition, options, operationContext);
isDone = true;
}
else
{
//We are not sure if reconfiguration is happening or not. We execute put in slow mode.
using (CloudBlobLease lease = new CloudBlobLease(configuration.Name, LeaseTakingPolicy.TryOnce))
{
if (lease.HasLease)
{
configuration.SyncWithCloud(ClientRegistry.GetConfigurationAccount());
DoDelete(deleteSnapshotsOption, accessCondition, options, operationContext);
isDone = true;
}
}
}
}
catch (StorageException ex)
{
throw ex;
}
catch (Exception ex)
{
throw ex;
}
} while (!isDone);
}
public static void PingAllServers()
{
List <string> allServers = config.GetServers();
Stopwatch watch = new Stopwatch();
for (int pingCount = 0; pingCount < 5; pingCount++)
{
foreach (string server in allServers)
{
CloudBlobClient blobClient = ClientRegistry.GetCloudBlobClient(server);
if (blobClient != null)
{
CloudBlobContainer blobContainer = blobClient.GetContainerReference(containerName);
//Log("Pinging " + SiteName(server) + " aka " + server + "...");
watch.Restart();
//we perform a dummy operation to get the rtt latency!
try
{
bool ok = blobContainer.Exists();
}
catch (StorageException se)
{
Log("Storage exception when pinging " + SiteName(server) + ": " + se.Message);
}
long el = watch.ElapsedMilliseconds;
ServerState ss = container.Monitor.GetServerState(server);
ss.AddRtt(el);
//Log("Pinged " + SiteName(server) + " in " + el + " milliseconds");
}
else
{
Log("Failed to ping " + SiteName(server));
}
}
}
}
/// <summary>
/// Perform read optimistically and then verify configuration.
/// </summary>
/// <param name="op">The read operation</param>
/// <param name="blob">The blob being read</param>
/// <param name="ss">The chosen server</param>
/// <returns>whether the read succeeded; if not, then it should be tried again.</returns>
private void SlowRead(ReadOp op)
{
// TODO: Deal with the case that we try to read from a replica that is no longer a replica and the read fails
// In this case, the read should be retried after refreshing the configuration.
ServerState ss = null;
try
{
bool isDone = false;
do // until we enter fast mode or succeed at reading in slow mode with the correct configuration
{
ss = slaEngine.FindServerToRead(blobName);
blobForRead = ClientRegistry.GetCloudBlob(ss.Name, containerName, blobName);
// TODO: this should really check if the reader wants strong consistency
// It could be that eventual consistency is fine but the SLA Engine just happened to choose a primary server
// In this case, we can do a fast mode read since we don't care if the chosen server is no longer a primary
if (configuration.IsInFastMode() || !configuration.PrimaryServers.Contains(ss.Name))
{
// it is fine to read from the selected secondary replica
// or from a primary replica if we have now entered fast mode
FastRead(op);
isDone = true;
}
else
{
// read from the selected replica that we believe to be a primary replica
watch.Start();
op(blobForRead);
watch.Stop();
// then see if the configuration has changed and the selected replica is no longer primary
configuration.SyncWithCloud(ClientRegistry.GetConfigurationAccount());
// TODO: check the epoch number on the configuration to make sure that we have not missed a configuration.
// i.e. we need to make sure that the server from which we read did not become a secondary during our read,
// and then back to primary when we checked.
// TODO: maybe we should check that the read delivered a non-zero utility.
// It is possible that the configuration was so stale that a much better server could have been chosen.
if (configuration.PrimaryServers.Contains(ss.Name))
{
//We have contacted the primary replica, hence we are good to go.
slaEngine.RecordObjectRead(blobForRead.Name, Timestamp(blobForRead), ss, watch.ElapsedMilliseconds);
isDone = true;
}
isDone = false; // not done
}
} while (!isDone);
}
// TODO: decide if we need to catch any exceptions here or just let then propagate through
catch (StorageException se)
{
if (StorageExceptionCode.NotFound(se))
{
//blob is not found.
//this is fine because the replica might have removed.
//it simply returns, so client need to re-execute if this happens.
//We can also re-execute the read here, but for debugging purposes, it's simpler for the client to re-execute.
//Of course in real system, we need to re-execute at this level.
return;
}
// storage exceptions are passed through to the caller
throw;
}
catch (Exception ex)
{
if (ex.Message.Contains("Object reference not set to an instance of an object"))
{
return;
}
throw ex;
}
}
public override double ComputeCost()
{
double result = 0;
if (NumberOfAddingReplica() == 0)
{
int secondarySyncPeriod = Configuration.GetSyncPeriod(ServerName);
result = CostModel.GetPrimaryTransactionalCost(numberOfReads, numberOfWrites) - (CostModel.GetSecondaryTransactionalCost(numberOfReads) + CostModel.GetSyncCost(numberOfWrites, secondarySyncPeriod));
}
else
{
result = CostModel.GetPrimaryTransactionalCost(numberOfReads, numberOfWrites) + CostModel.GetStorageCost(ClientRegistry.GetMainPrimaryContainer(Configuration.Name));
}
return(result);
}
public override double ComputeCost()
{
double result = 0;
DowngradePrimary p = new DowngradePrimary(Configuration.Name, ServerName, 0, null, ConfigurationActionSource.Constraint, numberOfReads, numberOfWrites);
if (Configuration.SecondaryServers.Contains(ServerName))
{
//all primaries will become secondary,
result = Configuration.PrimaryServers.Count * p.ComputeCost();
//a secondary will become primary:
int secondarySyncPeriod = Configuration.GetSyncPeriod(ServerName);
result += CostModel.GetPrimaryTransactionalCost(numberOfReads, numberOfWrites) - (CostModel.GetSecondaryTransactionalCost(numberOfReads) + CostModel.GetSyncCost(numberOfWrites, secondarySyncPeriod));
}
else if (Configuration.PrimaryServers.Contains(ServerName))
{
//all primaries except one will become secondary
result = (Configuration.PrimaryServers.Count - 1) * p.ComputeCost();
}
else
{
// all primaries will become secondary
result = Configuration.PrimaryServers.Count * p.ComputeCost();
//one non-replica becomes primary
result += CostModel.GetPrimaryTransactionalCost(numberOfReads, numberOfWrites) + CostModel.GetStorageCost(ClientRegistry.GetMainPrimaryContainer(Configuration.Name));
}
return(result);
}
public static void Main(string[] args)
{
Process currProc = Process.GetCurrentProcess();
containerName = "testcontainer"; //Guid.NewGuid().ToString("N").ToLower();
if (args.Length == 0)
{
args = new string[7];
// the number of test iterations
args[0] = "1";
// the number of blobs
args[1] = "10";
// the number of concurrent test workers.
args[2] = "2";
// the blob size in KB
args[3] = "1024";
// the number of parallel requests per blob
args[4] = "1";
// use https or not
args[5] = "false";
// the result folder name
args[6] = "folder1";
}
iterations = Int32.Parse(args[0]);
blobs = Int32.Parse(args[1]);
concurrency = Int32.Parse(args[2]);
blobSizeInKB = Int32.Parse(args[3]);
parallelRequestsPerBlob = Int32.Parse(args[4]);
useHttps = bool.Parse(args[5]);
resultFileFolderName = args[6];
if (!Directory.Exists(resultFileFolderName))
{
Directory.CreateDirectory(resultFileFolderName);
}
resultFile = string.Format(@"{6}\{0}_{1}_{2}_{3}_{4}_{5}.csv", blobSizeInKB, parallelRequestsPerBlob, concurrency, useHttps, iterations, blobs, resultFileFolderName);
ThreadPool.SetMinThreads(concurrency * parallelRequestsPerBlob, concurrency * parallelRequestsPerBlob);
accounts = Account.GetStorageAccounts(useHttps);
ClientRegistry.Init(accounts, accounts[configAccountName]);
configuration = new ReplicaConfiguration(containerName);
ClientRegistry.AddConfiguration(configuration);
if (firstClient)
{
// delete configuration blob and tables
// ReplicaConfiguration.DeleteConfiguration(containerName);
ConfigurationCloudStore backingStore = new ConfigurationCloudStore(accounts[configAccountName], configuration);
backingStore.DeleteConfiguration();
CloudTableClient ConfigurationCloudTableClient = accounts[configAccountName].CreateCloudTableClient();
CloudTable slaTable = ConfigurationCloudTableClient.GetTableReference(ConstPool.SLA_CONFIGURATION_TABLE_NAME);
slaTable.DeleteIfExists();
slaTable = ConfigurationCloudTableClient.GetTableReference(ConstPool.SESSION_STATE_CONFIGURATION_TABLE_NAME);
slaTable.DeleteIfExists();
Console.WriteLine("removed everything, wait 40 seconds ...");
Thread.Sleep(40000);
// recreate configuration
configuration.PrimaryServers.Add("dbtsouthstorage");
configuration.SyncWithCloud(accounts[configAccountName], false);
Console.WriteLine("recreated configuration, wait 10 seconds ...");
Thread.Sleep(10000);
}
else
{
// retrieve configuration from cloud
configuration.SyncWithCloud(accounts[configAccountName]);
}
if (firstClient)
{
slaEngine = new ConsistencySLAEngine(CreateShoppingCartSla1(), configuration);
}
else
{
slaEngine = new ConsistencySLAEngine(CreateShoppingCartSla2(), configuration);
}
blobClient = accounts["dbtsouthstorage"].CreateCloudBlobClient();
blobContainer = blobClient.GetContainerReference(containerName);
blobContainer.CreateIfNotExists();
capContainer = new CapCloudBlobContainer(blobContainer);
// Generate random data
BlobDataBuffer = new byte[1024 * blobSizeInKB];
Random random = new Random();
random.NextBytes(BlobDataBuffer);
//ServiceLevelAgreement sla = CreateShoppingCartSla();
Stopwatch totalWatch = new Stopwatch();
totalWatch.Start();
for (int m = 0; m < concurrency; m++)
{
ThreadPool.QueueUserWorkItem((o) =>
{
Interlocked.Increment(ref concurrentWorkers);
for (int i = 0; i < iterations; i++)
{
Console.WriteLine("Running thread " + m + "." + i);
Console.WriteLine("concurrent workers: " + concurrentWorkers);
try
{
// do upload blob test.
var blobsList = UploadBlob();
Console.WriteLine("Upload Finished ...\n");
// GET and DELETE.
DoGetAndDelete(blobsList);
Console.WriteLine("DoGetAndDelete Finished ...\n");
configure("client", containerName);
Console.WriteLine("Configure Finished ...\n");
}
catch (Exception e)
{
Console.WriteLine(e.ToString());
}
}
Interlocked.Decrement(ref concurrentWorkers);
});
}
Console.WriteLine("Program: Started to sleep");
Thread.Sleep(5000);
while (Interlocked.CompareExchange(ref concurrentWorkers, -1, 0) != -1)
{
if (concurrentWorkers < 0)
{
break;
}
Console.WriteLine("Waiting for a thread because there are " + concurrentWorkers + " threads.");
Thread.Sleep(5000);
}
Console.WriteLine("Finished execution. ");
ClientRegistry.GetConfigurationContainer(containerName).Delete();
blobContainer.DeleteIfExists();
totalWatch.Stop();
long totalTimeTaken = totalWatch.ElapsedMilliseconds;
using (StreamWriter sw = new StreamWriter(resultFile))
{
sw.Write(String.Format("Total time taken to run the test in ms : {0}\n\n", totalTimeTaken));
sw.Write(String.Format("Args:Concurrency: {0} BlobSizeInKB:{1} ParallelRequestsPerBlob:{2} UsingHttps:{3} \n", concurrency, blobSizeInKB, parallelRequestsPerBlob, useHttps));
// display result
DisplayResults(sw, "Insert", insertTimes);
DisplayResults(sw, "Get", getTimes);
DisplayResults(sw, "Delete", deleteTimes);
float tmp = 0;
foreach (SubSLA s in capContainer.SLA)
{
tmp += s.Utility * s.NumberOfHits;
}
sw.Write(String.Format("Current utility ", tmp));
// Display perf results
PerfMetrics metrics = new PerfMetrics();
metrics.Update(currProc);
metrics.Print(sw);
}
Console.Read();
}
private void MultiWrite(WriteOp op, AccessCondition access, List <SessionState> sessions = null, ServerMonitor monitor = null)
{
// This protocol uses three phases and ETags
// It assumes that the client is in fast mode and remains so throughout the protocol
// i.e. it assumes that the set of primaries does not change.
// TODO: recover from failed clients that may leave a write partially completed
ICloudBlob blob;
Dictionary <string, string> eTags;
// Phase 1. Mark intention to write with write-in-progress flags
eTags = SetWiPFlags();
if (eTags == null)
{
// flags were not successfully set, so abort the protocol
return;
}
// Phase 2. Perform write at all primaries
bool didAtLeastOne = false;
foreach (string server in configuration.PrimaryServers)
{
blob = ClientRegistry.GetCloudBlob(server, configuration.Name, blobName, false);
access.IfMatchETag = eTags[server];
watch.Start();
try
{
op(blob);
}
catch (StorageException)
{
// If writing fails at some primary, then abort the protocol
// It could be that a concurrent writer is in progress
// Note that some writes may have already been performed
// If so, then we leave the WiP flags set so the recovery process will kick in
// or so a concurrent writer can complete its protocol and overwrite our writes
// If not, then we can clear the WiP flags
if (!didAtLeastOne)
{
ClearWiPFlags(eTags);
}
return;
}
watch.Stop();
eTags[server] = blob.Properties.ETag;
didAtLeastOne = true;
// update session and server state
ServerState ss = (monitor == null) ? slaEngine.Monitor.GetServerState(server) : monitor.GetServerState(server);
ss.AddRtt(watch.ElapsedMilliseconds);
if (sessions == null)
{
slaEngine.Session.RecordObjectWritten(blobName, Timestamp(blob), ss);
}
else
{
foreach (SessionState session in sessions)
{
session.RecordObjectWritten(blobName, Timestamp(blob), ss);
}
}
}
// Phase 3. Clear write-in-progress flags to indicate that write has completed
ClearWiPFlags(eTags);
}
/// <summary>
/// Initializes the client registry with no client.
/// </summary>
static void InitializeClientRegistry()
{
Logger.Log("Initializing Client Registry ...", Color.blue, "NetworkServer");
_clientRegistry = new ClientRegistry();
Logger.Log("Client Registry Successfully Initialized.", Color.black, "NetworkServer");
}
