public LocationConstraint(string containerName, ReplicaConfiguration currentConfig, string serverName, LocationConstraintType type)
: base(containerName, currentConfig)
{
this.Type = type;
this.ConstrainedContainer = ClientRegistry.GetCloudBlobContainer(serverName, containerName);
this.ServerName = serverName;
}
public override ReplicaConfiguration IfApply(ReplicaConfiguration current)
{
List <string> primary = new List <string>();
List <string> secondary = new List <string>();
current.SecondaryServers.ForEach(s => { if (s != ServerName)
{
secondary.Add(s);
}
else
{
primary.Add(s);
} });
current.PrimaryServers.ForEach(s => { if (s == ServerName)
{
primary.Add(s);
}
else
{
secondary.Add(s);
} });
current.NonReplicaServers.ForEach(s => { if (s == ServerName)
{
primary.Add(s);
}
});
ReplicaConfiguration newConfig = new ReplicaConfiguration(current.Name, primary, secondary, null, null, false, true);
return(newConfig);
}
public ConsistencySLAEngine(ServiceLevelAgreement sla, ReplicaConfiguration config, SessionState sessionState = null, ServerMonitor monitor = null, ChosenUtility chosenUtility = null)
{
this.Sla = sla;
this.Config = config;
if (sessionState != null)
{
this.Session = sessionState;
}
else
{
this.Session = new SessionState();
}
if (monitor != null)
{
this.Monitor = monitor;
}
else
{
this.Monitor = new ServerMonitor(config);
}
this.chosenUtility = chosenUtility;
this.selector = new ServerSelector(Session, Config, Monitor);
}
/// <summary>
/// Creates a new instance of a CapCloudBlob.
/// </summary>
/// <param name="strong">A reference to the strongly consistent copy of the blob.</param>
/// <param name="eventual">A reference to the eventually consistent copy of the blob.</param>
/// <param name="engine">The SLA enforcement engine.</param>
public CapCloudBlobOld(string name, ReplicaConfiguration configuration, ConsistencySLAEngine engine)
{
this.Name = name;
this.slaEngine = engine;
this.configuration = configuration;
this.watch = new Stopwatch();
this.protocol = new ReadWriteFramework(name, configuration, engine);
}
public ReadWriteFramework(string blobName, ReplicaConfiguration configuration, ConsistencySLAEngine engine)
{
this.blobName = blobName;
this.containerName = configuration.Name;
this.configuration = configuration;
this.slaEngine = engine;
this.blobForRead = null;
this.watch = new Stopwatch();
}
public static void Initialize()
{
// get list of Azure storage accounts
accounts = PileusApp.Account.GetStorageAccounts(false);
ClientRegistry.Init(accounts, accounts[configStorageSite]);
// delete cloud configuration if desired
if (clearConfiguration)
{
ReplicaConfiguration configToDelete = new ReplicaConfiguration(containerName);
ConfigurationCloudStore backingStore = new ConfigurationCloudStore(accounts[configStorageSite], configToDelete);
Log("Deleting configuration in Azure...");
backingStore.DeleteConfiguration();
Log("Done. Now waiting for it to really take effect...");
Thread.Sleep(40000); // give system a chance to complete delete
}
// read/create configuration
Log("Creating/reading configuration...");
config = new ReplicaConfiguration(containerName, PrimaryServers, SecondaryServers, NonReplicaServers, ReadOnlySecondaryServers, cloudBackedConfiguration, stableConfiguration);
ClientRegistry.AddConfiguration(config);
// upload data into containers if desired
if (reloadDatabase)
{
CreateDatabase(numBlobs);
}
// create SLAs
slas = new Dictionary <string, ServiceLevelAgreement>();
slas["strong"] = CreateConsistencySla(Consistency.Strong);
slas["causal"] = CreateConsistencySla(Consistency.Causal);
slas["bounded"] = CreateBoundedConsistencySla(boundInSeconds);
slas["readmywrites"] = CreateConsistencySla(Consistency.ReadMyWrites);
slas["monotonic"] = CreateConsistencySla(Consistency.MonotonicReads);
slas["eventual"] = CreateConsistencySla(Consistency.Eventual);
currentSLA = CreateFastOrStrongSla();
slas["sla"] = currentSLA;
// get/create replicated container
Log("Creating replicated container...");
Dictionary <string, CloudBlobContainer> containers = new Dictionary <string, CloudBlobContainer>();
foreach (string site in accounts.Keys)
{
CloudBlobClient blobClient = accounts[site].CreateCloudBlobClient();
CloudBlobContainer blobContainer = blobClient.GetContainerReference(containerName);
blobContainer.CreateIfNotExists();
containers.Add(site, blobContainer);
}
container = new CapCloudBlobContainer(containers, PrimaryServers.First());
container.Configuration = config;
container.Monitor = new ServerMonitor(config);
configurator = new Configurator(containerName);
}
public ReplicationFactorConstraint(string containerName, ReplicaConfiguration currentConfig, int minReplicationFactor, int MaxReplicationFactor)
: base(containerName, currentConfig)
{
this.MinReplicationFactor = minReplicationFactor;
this.MaxReplicationFactor = MaxReplicationFactor;
this.containerName = containerName;
if (MaxReplicationFactor < minReplicationFactor)
{
throw new ArgumentException("Minimum is greater than maximum!");
}
}
public override ReplicaConfiguration IfApply(ReplicaConfiguration current)
{
List <string> primary = new List <string>();
List <string> secondary = new List <string>();
current.SecondaryServers.ForEach(s => { if (s != ServerName)
{
secondary.Add(s);
}
});
current.PrimaryServers.ForEach(s => { primary.Add(s); });
return(new ReplicaConfiguration(current.Name, primary, secondary));
}
/// <summary>
/// Lease configuration container responsible for this blob along with blobs in the containers
/// </summary>
/// <param name="blobName">Particular blobs we would like to lease</param>
/// <param name="containerElementSet">list of containers</param>
/// <param name="putOptimization">perform put optimization. I.e., for put operations, it is not required to take lease if there is only one primary.</param>
public PrimaryCloudBlobLease(string blobName, ReplicaConfiguration configuration, bool putOptimization = false)
{
if ((putOptimization) && configuration.PrimaryServers.Count == 1)
{
//There is only one primary. No need to
HasLease = true;
return;
}
leasedBlobs = new Dictionary <ICloudBlob, string>();
//Reconfiguration is not going to happen in near future.
//We can safely take leases of primary blobs.
this.ProposedLeaseId = Guid.NewGuid().ToString();
foreach (string serverName in configuration.PrimaryServers)
{
try
{
ICloudBlob blob = ClientRegistry.GetCloudBlob(serverName, configuration.Name, blobName);
if (!blob.Exists())
{
//we cannot take lease on a non-existing blob.
//Hence, we create it first.
byte[] dummy = new byte[1];
dummy[0] = (byte)0;
var ms = new MemoryStream(dummy);
blob.UploadFromStream(ms, null, null, null);
}
string leaseID = blob.AcquireLease(null, ProposedLeaseId);
leasedBlobs.Add(blob, leaseID);
}
catch (StorageException ex)
{
releaseContainers();
leasedBlobs.Clear();
HasLease = false;
if (ex.GetBaseException().Message.Contains("409")) //Conflict
{
return;
}
else
{
throw ex;
}
}
}
HasLease = true;
}
public static string PrintCurrentConfiguration()
{
string result = null;
ReplicaConfiguration config = ClientRegistry.GetConfiguration(containerName, false);
result += "Current configuration for " + config.Name + ":" + "\r\n";
result += "Primary: ";
bool first = true;
foreach (string name in config.PrimaryServers)
{
if (first)
{
first = false;
}
else
{
result += ", ";
}
result += SiteName(name);
}
;
result += "\r\n";
result += "Secondaries: ";
first = true;
foreach (string name in config.SecondaryServers)
{
if (first)
{
first = false;
}
else
{
result += ", ";
}
result += SiteName(name);
}
;
result += "\r\n";
return(result);
}
private void buttonGetLatency_Click(object sender, EventArgs e)
{
ServerMonitor monitor = DemoLib.GetServerMonitor();
ReplicaConfiguration config = DemoLib.GetCurrentConfiguration();
List <string> allServers = config.GetServers();
List <string> siteNames = new List <string>();
List <string> siteLatency = new List <string>();
foreach (string server in allServers)
{
ServerState ss = monitor.GetServerState(server);
siteNames.Add(DemoLib.SiteName(server));
siteLatency.Add(ss.RTTs.FindAverage().ToString("F0").PadLeft(5));
}
listBoxLatencySites.DataSource = null;
listBoxLatencySites.DataSource = siteNames;
listBoxLatencySites.ClearSelected();
listBoxLatencyTimes.DataSource = null;
listBoxLatencyTimes.DataSource = siteLatency;
listBoxLatencyTimes.ClearSelected();
}
protected override void ProcessRecord()
{
ReplicaConfiguration config = new ReplicaConfiguration
{
FailoverTarget = FailoverTarget,
MysqlReplicaConfiguration = new MySqlReplicaConfiguration
{
CaCertificate = MySqlCaCert,
ClientCertificate = MySqlClientCert,
ClientKey = MySqlClientKey,
ConnectRetryInterval = MySqlRetryInterval,
DumpFilePath = MySqlDumpPath,
MasterHeartbeatPeriod = MySqlHeartbeatPeriod,
Password = MySqlPassword,
SslCipher = MySqlSslCipher,
Username = MySqlUser,
VerifyServerCertificate = MySqlVerifyCertificate,
}
};
WriteObject(config);
}
/// <summary>
/// Constructs a new monitor to keep track of the state of each server and optionally to ping servers periodically.
/// </summary>
/// <param name="config">The configuration of servers</param>
/// <param name="periodicPing">Whether to ping servers periodically</param>
public ServerMonitor(ReplicaConfiguration config, bool periodicPing = false)
{
this.replicas = new Dictionary <string, ServerState>();
this.configuration = config;
foreach (string primary in config.PrimaryServers)
{
// Note that the configuration may change later, and so clients should view the isPrimary bit as simply a hint
RegisterServer(primary, true, 1);
}
foreach (string secondary in config.SecondaryServers)
{
RegisterServer(secondary, false, 2);
}
foreach (string nonreplica in config.NonReplicaServers)
{
RegisterServer(nonreplica, false, 3);
}
if (periodicPing)
{
periodicPingTask = Task.Factory.StartNew(() => PeriodPing());
}
}
private static HashSet <LatencyDistribution> GetApplicableServers(SubSLA s, ReplicaConfiguration config, ClientUsageData usage)
{
HashSet <LatencyDistribution> result = new HashSet <LatencyDistribution>();
foreach (string name in config.PrimaryServers)
{
if (usage.ServerRTTs.ContainsKey(name))
{
result.Add(usage.ServerRTTs[name]);
}
}
if (s.Consistency != Consistency.Strong)
{
foreach (string name in config.SecondaryServers)
{
if (usage.ServerRTTs.ContainsKey(name))
{
result.Add(usage.ServerRTTs[name]);
}
}
}
return(result);
}
public static void CheckAction(ConfigurationAction action, ServiceLevelAgreement SLA, ClientUsageData usage, ReplicaConfiguration currentConfig)
{
ReplicaConfiguration newConfig = action.IfApply(currentConfig);
foreach (SubSLA s in SLA)
{
float currentProbability = GetApplicableServers(s, currentConfig, usage).Max(server => server.ProbabilityOfFindingValueLessThanGiven(s.Latency));
//probability of new state should be computed in pessimistic mode. I.e., if there is no entry in the distribution of session state, instead of returning 1, we return 0.
float newProbability = GetApplicableServers(s, newConfig, usage).Max(server => server.ProbabilityOfFindingValueLessThanGiven(s.Latency, false));
/*
* // Not sure why the utility is multiplied by hits or misses
* if (newProbability > currentProbability)
* {
* action.AddUtility((newProbability - currentProbability) * s.NumberOfMisses * s.Utility);
* }
* else
* {
* action.AddUtility((newProbability - currentProbability) * s.NumberOfHits * s.Utility);
* }
* */
action.AddUtility((newProbability - currentProbability) * s.Utility);
}
}
/// <summary> /// returns a new configuration by applying the action to the input current configuration. /// </summary> /// <param name="current">The current configuration</param> /// <returns></returns> public abstract ReplicaConfiguration IfApply(ReplicaConfiguration current);
private static List <ConfigurationAction> GetActionsForEventualConsistency(string ContainerName, ReplicaConfiguration config, SubSLA subSLA, ClientUsageData usage, string slaId)
{
List <ConfigurationAction> result = new List <ConfigurationAction>();
int desiredLatency = subSLA.Latency;
// we first compute the maximum probability of meeting the latency with current replicas.
float currentMaxProbability = usage.ServerRTTs
.Where(sr => config.PrimaryServers.Contains(sr.Key) || config.SecondaryServers.Contains(sr.Key))
.Max(sr => sr.Value.ProbabilityOfFindingValueLessThanGiven(desiredLatency));
// for any non-replica with higher probabiliy than both the currentMaxProbability and MIN_ACCEPTABLE_PROB_FOR_EVENTUAL_CONS, we create a new action.
IEnumerable <string> chosenServers = usage.ServerRTTs
.Where(sr => config.NonReplicaServers.Contains(sr.Key) &&
sr.Value.ProbabilityOfFindingValueLessThanGiven(desiredLatency) > currentMaxProbability &&
sr.Value.ProbabilityOfFindingValueLessThanGiven(desiredLatency) > ConstPool.MIN_ACCEPTABLE_PROB_FOR_CONS)
.Select(sr => sr.Key);
foreach (string server in chosenServers)
{
float utilityGain = (usage.ServerRTTs[server].ProbabilityOfFindingValueLessThanGiven(desiredLatency) - currentMaxProbability) * subSLA.Utility; /* * subSLA.NumberOfMisses*/
result.Add(new AddSecondaryServer(ContainerName, server, utilityGain, slaId, ConfigurationActionSource.NonConstraint, numberOfReads, numberOfWrites));
}
return(result);
}
public ConfigurationConstraint(string containerName, ReplicaConfiguration currentConfig)
{
this.Configuration = currentConfig;
}
private static List <ConfigurationAction> GetActionsFor_RMW_MON_CAUSAL_BOUNDED_Consistency(string ContainerName, ReplicaConfiguration config, SubSLA subSLA, ClientUsageData usage, string slaId)
{
List <ConfigurationAction> result = new List <ConfigurationAction>();
int desiredLatency = subSLA.Latency;
float currentMaxProbability = 0;
string nearestServer = null;
// Find the closest replica, i.e. the one that has the maximum probability of meeting the desired latency
foreach (string server in usage.ServerRTTs.Keys)
{
if (config.PrimaryServers.Contains(server) || config.SecondaryServers.Contains(server))
{
float prob = usage.ServerRTTs[server].ProbabilityOfFindingValueLessThanGiven(desiredLatency);
if (prob > currentMaxProbability)
{
currentMaxProbability = prob;
nearestServer = server;
}
}
}
// One action is to reduce the sync period for the nearest replica.
if (nearestServer != null && !config.PrimaryServers.Contains(nearestServer))
{
int oldInterval = config.GetSyncPeriod(nearestServer);
if ((oldInterval * ConstPool.ADJUSTING_SYNC_INTERVAL_MULTIPLIER) > ConstPool.MINIMUM_ALLOWED_SYNC_INTERVAL)
{
// TODO: This can't be right. The utility gain for adjusting the sync period is hard to predict.
float utilityGain = currentMaxProbability * subSLA.Utility; /* * subSLA.NumberOfMisses*/
result.Add(new AdjustSyncPeriod(ContainerName, nearestServer, utilityGain, slaId, ConfigurationActionSource.NonConstraint, numberOfReads, numberOfWrites));
}
}
// For any non-replica with higher probabiliy than both the currentMaxProbability and MIN_ACCEPTABLE_PROB_FOR_EVENTUAL_CONS, we create a new action.
IEnumerable <string> chosenServers = usage.ServerRTTs
.Where(sr => config.NonReplicaServers.Contains(sr.Key) &&
sr.Value.ProbabilityOfFindingValueLessThanGiven(desiredLatency) > currentMaxProbability &&
sr.Value.ProbabilityOfFindingValueLessThanGiven(desiredLatency) > ConstPool.MIN_ACCEPTABLE_PROB_FOR_CONS)
.Select(sr => sr.Key);
foreach (string server in chosenServers)
{
float utilityGain = (usage.ServerRTTs[server].ProbabilityOfFindingValueLessThanGiven(desiredLatency) - currentMaxProbability) * subSLA.Utility; /* * subSLA.NumberOfMisses*/
result.Add(new AddSecondaryServer(ContainerName, server, utilityGain, slaId, ConfigurationActionSource.NonConstraint, numberOfReads, numberOfWrites));
}
return(result);
}
public FixedPrimaryConstraint(string containerName, ReplicaConfiguration currentConfig)
: base(containerName, currentConfig)
{
this.containerName = containerName;
}
/// <summary>
/// Computes new actions to improve the utility of the given SLA
/// </summary>
/// <param name="ContainerName"></param>
/// <param name="SLA"></param>
/// <param name="state"></param>
/// <returns></returns>
public static Dictionary <string, ConfigurationAction> ComputeActions(string ContainerName, ReplicaConfiguration currentConfig, ServiceLevelAgreement SLA, ClientUsageData usage)
{
numberOfReads = usage.NumberOfReads;
numberOfWrites = usage.NumberOfWrites;
Dictionary <string, ConfigurationAction> result = new Dictionary <string, ConfigurationAction>();
foreach (SubSLA s in SLA)
{
if (s.NumberOfMisses == 0)
{
return(result);
}
if (s.Consistency == Consistency.Strong)
{
GetActionsForStrongConsistency(ContainerName, currentConfig, s, usage, SLA.Id).ForEach(i => { if (!result.ContainsKey(i.Id))
{
result[i.Id] = i;
}
else
{
result[i.Id].Merge(i);
} });
}
else if (s.Consistency == Consistency.Eventual)
{
GetActionsForEventualConsistency(ContainerName, currentConfig, s, usage, SLA.Id).ForEach(i => { if (!result.ContainsKey(i.Id))
{
result[i.Id] = i;
}
else
{
result[i.Id].Merge(i);
} });
}
else
{
GetActionsFor_RMW_MON_CAUSAL_BOUNDED_Consistency(ContainerName, currentConfig, s, usage, SLA.Id).ForEach(i => { if (!result.ContainsKey(i.Id))
{
result[i.Id] = i;
}
else
{
result[i.Id].Merge(i);
} });
}
}
return(result);
}
public CostConstraint(string containerName, ReplicaConfiguration currentConfig, double costLimit)
: base(containerName, currentConfig)
{
this.costLimit = costLimit;
}
public static void Main(string[] args)
{
Console.CancelKeyPress += new ConsoleCancelEventHandler(Cancelled);
if (args.Length == 0)
{
args = new string[7];
//storage account locating configuration of given container
args[0] = "dbtsouthstorage"; // "devstoreaccount1";
// container name
args[1] = "testcontainer";
// the result folder name
args[2] = "folder1";
//sleep time between ticks in milliseconds.
args[3] = "90000";
//interval between configuration
args[4] = "2";
//duration of experiment in ticks
args[5] = "24";
//start tick for configuration
args[6] = "1";
}
configurationSite = args[0];
containerName = args[1];
resultFileFolderName = args[2];
sleepTimeBetweenTicks = Int32.Parse(args[3]);
ticksBetweenConfigurations = Int32.Parse(args[4]);
experimentDurationInTicks = Int32.Parse(args[5]);
startTickOfConfiguration = Int32.Parse(args[6]);
Dictionary <string, CloudStorageAccount> acounts = Account.GetStorageAccounts(true);
acounts.Remove("devstoreaccount1");
ClientRegistry.Init(acounts, Account.GetStorageAccounts(true)[configurationSite]);
ReplicaConfiguration configuration = ClientRegistry.GetConfiguration(containerName, false);
Configurator conf = new Configurator(containerName);
#region replicator
Replicator replicator = new Replicator(containerName);
replicator.Start();
#endregion
#region configurator
List <ConfigurationConstraint> constraints = new List <ConfigurationConstraint>();
//constraints.Add(new LocationConstraint(containerName, "dbteastasiastorage", LocationConstraintType.Replicate));
constraints.Add(new ReplicationFactorConstraint(containerName, configuration, 1, 2));
DateTime startTime = DateTime.Now;
Thread.Sleep(startTickOfConfiguration * sleepTimeBetweenTicks);
while (DateTime.Now.Subtract(startTime).TotalMilliseconds < (experimentDurationInTicks * sleepTimeBetweenTicks))
{
try
{
configuration = ClientRegistry.GetConfiguration(containerName, false);
Console.WriteLine("Starting to reconfigure. Current Epoch: " + configuration.Epoch);
conf.Configure(ClientRegistry.GetConfigurationAccount(), configuration.Epoch, configuration, constraints);
Console.WriteLine(Configurator.Logs);
Console.WriteLine(">>>>>>>>>>> Finished. Current Epoch: " + configuration.Epoch + "<<<<<<<<<<<<<<<<");
}
catch (Exception ex)
{
Console.WriteLine(ex.StackTrace);
}
Thread.Sleep(ticksBetweenConfigurations * sleepTimeBetweenTicks);
}
#endregion
return;
}
private void setReplicasButton_Click(object sender, EventArgs e)
{
ReplicaConfiguration config = DemoLib.GetCurrentConfiguration();
config.PrimaryServers.Clear();
config.SecondaryServers.Clear();
config.NonReplicaServers.Clear();
string server = DemoLib.ServerName("West US");
if (radioButtonWestUSPrimary.Checked)
{
config.PrimaryServers.Add(server);
}
else if (radioButtonWestUSSecondary.Checked)
{
config.SecondaryServers.Add(server);
}
else
{
config.NonReplicaServers.Add(server);
}
server = DemoLib.ServerName("East US");
if (radioButtonEastUSPrimary.Checked)
{
config.PrimaryServers.Add(server);
}
else if (radioButtonEastUSSecondary.Checked)
{
config.SecondaryServers.Add(server);
}
else
{
config.NonReplicaServers.Add(server);
}
server = DemoLib.ServerName("South US");
if (radioButtonSouthUSPrimary.Checked)
{
config.PrimaryServers.Add(server);
}
else if (radioButtonSouthUSSecondary.Checked)
{
config.SecondaryServers.Add(server);
}
else
{
config.NonReplicaServers.Add(server);
}
server = DemoLib.ServerName("North US");
if (radioButtonNorthUSPrimary.Checked)
{
config.PrimaryServers.Add(server);
}
else if (radioButtonNorthUSSecondary.Checked)
{
config.SecondaryServers.Add(server);
}
else
{
config.NonReplicaServers.Add(server);
}
server = DemoLib.ServerName("West Europe");
if (radioButtonWestEuropePrimary.Checked)
{
config.PrimaryServers.Add(server);
}
else if (radioButtonWestEuropeSecondary.Checked)
{
config.SecondaryServers.Add(server);
}
else
{
config.NonReplicaServers.Add(server);
}
server = DemoLib.ServerName("North Europe");
if (radioButtonNorthEuropePrimary.Checked)
{
config.PrimaryServers.Add(server);
}
else if (radioButtonNorthEuropeSecondary.Checked)
{
config.SecondaryServers.Add(server);
}
else
{
config.NonReplicaServers.Add(server);
}
server = DemoLib.ServerName("East Asia");
if (radioButtonAsiaPrimary.Checked)
{
config.PrimaryServers.Add(server);
}
else if (radioButtonAsiaSecondary.Checked)
{
config.SecondaryServers.Add(server);
}
else
{
config.NonReplicaServers.Add(server);
}
server = DemoLib.ServerName("Brazil");
if (radioButtonBrazilPrimary.Checked)
{
config.PrimaryServers.Add(server);
}
else if (radioButtonBrazilSecondary.Checked)
{
config.SecondaryServers.Add(server);
}
else
{
config.NonReplicaServers.Add(server);
}
initialConfig = false;
reconfigSampler = DemoLib.NewSampler();
DemoLib.SetCurrentConfiguration(config);
}
public static void SetCurrentConfiguration(ReplicaConfiguration newConfig)
{
config = newConfig;
slas["sla"].ResetHitsAndMisses();
}
private void getReplicasButton_Click(object sender, EventArgs e)
{
ReplicaConfiguration config = DemoLib.GetCurrentConfiguration();
string server = DemoLib.ServerName("West US");
if (config.PrimaryServers.Contains(server))
{
radioButtonWestUSPrimary.Checked = true;
}
else if (config.SecondaryServers.Contains(server))
{
radioButtonWestUSSecondary.Checked = true;
}
else
{
radioButtonWestUSUnused.Checked = true;
}
if (config.ReadOnlySecondaryServers.Contains(server))
{
radioButtonWestUSPrimary.Enabled = false;
}
server = DemoLib.ServerName("East US");
if (config.PrimaryServers.Contains(server))
{
radioButtonEastUSPrimary.Checked = true;
}
else if (config.SecondaryServers.Contains(server))
{
radioButtonEastUSSecondary.Checked = true;
}
else
{
radioButtonEastUSUnused.Checked = true;
}
if (config.ReadOnlySecondaryServers.Contains(server))
{
radioButtonEastUSPrimary.Enabled = false;
}
server = DemoLib.ServerName("South US");
if (config.PrimaryServers.Contains(server))
{
radioButtonSouthUSPrimary.Checked = true;
}
else if (config.SecondaryServers.Contains(server))
{
radioButtonSouthUSSecondary.Checked = true;
}
else
{
radioButtonSouthUSUnused.Checked = true;
}
if (config.ReadOnlySecondaryServers.Contains(server))
{
radioButtonSouthUSPrimary.Enabled = false;
}
server = DemoLib.ServerName("North US");
if (config.PrimaryServers.Contains(server))
{
radioButtonNorthUSPrimary.Checked = true;
}
else if (config.SecondaryServers.Contains(server))
{
radioButtonNorthUSSecondary.Checked = true;
}
else
{
radioButtonNorthUSUnused.Checked = true;
}
if (config.ReadOnlySecondaryServers.Contains(server))
{
radioButtonNorthUSPrimary.Enabled = false;
}
server = DemoLib.ServerName("West Europe");
if (config.PrimaryServers.Contains(server))
{
radioButtonWestEuropePrimary.Checked = true;
}
else if (config.SecondaryServers.Contains(server))
{
radioButtonWestEuropeSecondary.Checked = true;
}
else
{
radioButtonWestEuropeUnused.Checked = true;
}
if (config.ReadOnlySecondaryServers.Contains(server))
{
radioButtonWestEuropePrimary.Enabled = false;
}
server = DemoLib.ServerName("North Europe");
if (config.PrimaryServers.Contains(server))
{
radioButtonNorthEuropePrimary.Checked = true;
}
else if (config.SecondaryServers.Contains(server))
{
radioButtonNorthEuropeSecondary.Checked = true;
}
else
{
radioButtonNorthEuropeUnused.Checked = true;
}
if (config.ReadOnlySecondaryServers.Contains(server))
{
radioButtonNorthEuropePrimary.Enabled = false;
}
server = DemoLib.ServerName("East Asia");
if (config.PrimaryServers.Contains(server))
{
radioButtonAsiaPrimary.Checked = true;
}
else if (config.SecondaryServers.Contains(server))
{
radioButtonAsiaSecondary.Checked = true;
}
else
{
radioButtonAsiaUnused.Checked = true;
}
if (config.ReadOnlySecondaryServers.Contains(server))
{
radioButtonAsiaPrimary.Enabled = false;
}
server = DemoLib.ServerName("Brazil");
if (config.PrimaryServers.Contains(server))
{
radioButtonBrazilPrimary.Checked = true;
}
else if (config.SecondaryServers.Contains(server))
{
radioButtonBrazilSecondary.Checked = true;
}
else
{
radioButtonBrazilUnused.Checked = true;
}
if (config.ReadOnlySecondaryServers.Contains(server))
{
radioButtonBrazilPrimary.Enabled = false;
}
}
public override ReplicaConfiguration IfApply(ReplicaConfiguration current)
{
//return new SessionState(sessionState.GetSecondaryReplicaServers().Values.ToList(), sessionState.GetPrimaryReplicaServers().Values.ToList());
return(current);
}
public static void Main(string[] args)
{
Process currProc = Process.GetCurrentProcess();
containerName = "testcontainer"; //Guid.NewGuid().ToString("N").ToLower();
if (args.Length == 0)
{
args = new string[7];
// the number of test iterations
args[0] = "1";
// the number of blobs
args[1] = "10";
// the number of concurrent test workers.
args[2] = "2";
// the blob size in KB
args[3] = "1024";
// the number of parallel requests per blob
args[4] = "1";
// use https or not
args[5] = "false";
// the result folder name
args[6] = "folder1";
}
iterations = Int32.Parse(args[0]);
blobs = Int32.Parse(args[1]);
concurrency = Int32.Parse(args[2]);
blobSizeInKB = Int32.Parse(args[3]);
parallelRequestsPerBlob = Int32.Parse(args[4]);
useHttps = bool.Parse(args[5]);
resultFileFolderName = args[6];
if (!Directory.Exists(resultFileFolderName))
{
Directory.CreateDirectory(resultFileFolderName);
}
resultFile = string.Format(@"{6}\{0}_{1}_{2}_{3}_{4}_{5}.csv", blobSizeInKB, parallelRequestsPerBlob, concurrency, useHttps, iterations, blobs, resultFileFolderName);
ThreadPool.SetMinThreads(concurrency * parallelRequestsPerBlob, concurrency * parallelRequestsPerBlob);
accounts = Account.GetStorageAccounts(useHttps);
ClientRegistry.Init(accounts, accounts[configAccountName]);
configuration = new ReplicaConfiguration(containerName);
ClientRegistry.AddConfiguration(configuration);
if (firstClient)
{
// delete configuration blob and tables
// ReplicaConfiguration.DeleteConfiguration(containerName);
ConfigurationCloudStore backingStore = new ConfigurationCloudStore(accounts[configAccountName], configuration);
backingStore.DeleteConfiguration();
CloudTableClient ConfigurationCloudTableClient = accounts[configAccountName].CreateCloudTableClient();
CloudTable slaTable = ConfigurationCloudTableClient.GetTableReference(ConstPool.SLA_CONFIGURATION_TABLE_NAME);
slaTable.DeleteIfExists();
slaTable = ConfigurationCloudTableClient.GetTableReference(ConstPool.SESSION_STATE_CONFIGURATION_TABLE_NAME);
slaTable.DeleteIfExists();
Console.WriteLine("removed everything, wait 40 seconds ...");
Thread.Sleep(40000);
// recreate configuration
configuration.PrimaryServers.Add("dbtsouthstorage");
configuration.SyncWithCloud(accounts[configAccountName], false);
Console.WriteLine("recreated configuration, wait 10 seconds ...");
Thread.Sleep(10000);
}
else
{
// retrieve configuration from cloud
configuration.SyncWithCloud(accounts[configAccountName]);
}
if (firstClient)
{
slaEngine = new ConsistencySLAEngine(CreateShoppingCartSla1(), configuration);
}
else
{
slaEngine = new ConsistencySLAEngine(CreateShoppingCartSla2(), configuration);
}
blobClient = accounts["dbtsouthstorage"].CreateCloudBlobClient();
blobContainer = blobClient.GetContainerReference(containerName);
blobContainer.CreateIfNotExists();
capContainer = new CapCloudBlobContainer(blobContainer);
// Generate random data
BlobDataBuffer = new byte[1024 * blobSizeInKB];
Random random = new Random();
random.NextBytes(BlobDataBuffer);
//ServiceLevelAgreement sla = CreateShoppingCartSla();
Stopwatch totalWatch = new Stopwatch();
totalWatch.Start();
for (int m = 0; m < concurrency; m++)
{
ThreadPool.QueueUserWorkItem((o) =>
{
Interlocked.Increment(ref concurrentWorkers);
for (int i = 0; i < iterations; i++)
{
Console.WriteLine("Running thread " + m + "." + i);
Console.WriteLine("concurrent workers: " + concurrentWorkers);
try
{
// do upload blob test.
var blobsList = UploadBlob();
Console.WriteLine("Upload Finished ...\n");
// GET and DELETE.
DoGetAndDelete(blobsList);
Console.WriteLine("DoGetAndDelete Finished ...\n");
configure("client", containerName);
Console.WriteLine("Configure Finished ...\n");
}
catch (Exception e)
{
Console.WriteLine(e.ToString());
}
}
Interlocked.Decrement(ref concurrentWorkers);
});
}
Console.WriteLine("Program: Started to sleep");
Thread.Sleep(5000);
while (Interlocked.CompareExchange(ref concurrentWorkers, -1, 0) != -1)
{
if (concurrentWorkers < 0)
{
break;
}
Console.WriteLine("Waiting for a thread because there are " + concurrentWorkers + " threads.");
Thread.Sleep(5000);
}
Console.WriteLine("Finished execution. ");
ClientRegistry.GetConfigurationContainer(containerName).Delete();
blobContainer.DeleteIfExists();
totalWatch.Stop();
long totalTimeTaken = totalWatch.ElapsedMilliseconds;
using (StreamWriter sw = new StreamWriter(resultFile))
{
sw.Write(String.Format("Total time taken to run the test in ms : {0}\n\n", totalTimeTaken));
sw.Write(String.Format("Args:Concurrency: {0} BlobSizeInKB:{1} ParallelRequestsPerBlob:{2} UsingHttps:{3} \n", concurrency, blobSizeInKB, parallelRequestsPerBlob, useHttps));
// display result
DisplayResults(sw, "Insert", insertTimes);
DisplayResults(sw, "Get", getTimes);
DisplayResults(sw, "Delete", deleteTimes);
float tmp = 0;
foreach (SubSLA s in capContainer.SLA)
{
tmp += s.Utility * s.NumberOfHits;
}
sw.Write(String.Format("Current utility ", tmp));
// Display perf results
PerfMetrics metrics = new PerfMetrics();
metrics.Update(currProc);
metrics.Print(sw);
}
Console.Read();
}
private static List <ConfigurationAction> GetActionsForStrongConsistency(string ContainerName, ReplicaConfiguration config, SubSLA subSLA, ClientUsageData usage, string slaId)
{
List <ConfigurationAction> result = new List <ConfigurationAction>();
int desiredLatency = subSLA.Latency;
// we first compute the maximum probability of meeting the latency with primary servers.
float currentMaxProbability = usage.ServerRTTs
.Where(sr => config.PrimaryServers.Contains(sr.Key))
.Max(sr => sr.Value.ProbabilityOfFindingValueLessThanGiven(desiredLatency));
// for any secondary replica with higher probability than the primary, we create an action to make that secondary the solo primary.
// TODO: shouldn't this just pick one secondary with the max probability, i.e. why try to make multiple solo primaries?
IEnumerable <string> chosenSecondaries = usage.ServerRTTs
.Where(sr => config.SecondaryServers.Contains(sr.Key) &&
!config.ReadOnlySecondaryServers.Contains(sr.Key) &&
sr.Value.ProbabilityOfFindingValueLessThanGiven(desiredLatency) > currentMaxProbability)
.Select(sr => sr.Key);
foreach (string server in chosenSecondaries)
{
float utilityGain = (usage.ServerRTTs[server].ProbabilityOfFindingValueLessThanGiven(desiredLatency) - currentMaxProbability) * subSLA.Utility; /* * subSLA.NumberOfMisses*/
result.Add(new MakeSoloPrimaryServer(ContainerName, server, utilityGain, slaId, ConfigurationActionSource.NonConstraint, numberOfReads, numberOfWrites));
}
// for any server with higher probabiliy than the primary, we create an action to make that server one of the primaries.
IEnumerable <string> chosenServers = usage.ServerRTTs
.Where(sr => (config.SecondaryServers.Contains(sr.Key) || config.NonReplicaServers.Contains(sr.Key)) &&
!config.ReadOnlySecondaryServers.Contains(sr.Key) &&
sr.Value.ProbabilityOfFindingValueLessThanGiven(desiredLatency) > currentMaxProbability)
.Select(sr => sr.Key);
foreach (string server in chosenServers)
{
float utilityGain = (usage.ServerRTTs[server].ProbabilityOfFindingValueLessThanGiven(desiredLatency) - currentMaxProbability) * subSLA.Utility; /* * subSLA.NumberOfMisses*/
result.Add(new AddPrimaryServer(ContainerName, server, utilityGain, slaId, ConfigurationActionSource.NonConstraint, numberOfReads, numberOfWrites));
}
return(result);
}
/// <summary>
/// Constructs a new server selector instance.
/// </summary>
/// <param name="session">holds the session state</param>
/// <param name="config">holds the current replica configuration</param>
public ServerSelector(SessionState session, ReplicaConfiguration config, ServerMonitor monitor)
{
this.session = session;
this.config = config;
this.monitor = monitor;
}
