public async Task AzureGossip_ConfigGossip()
{
// start clean
await gossipTable.DeleteAllEntries();
// push empty data
await gossipTable.Publish(new MultiClusterData());
// push and pull empty data
var answer = await gossipTable.Synchronize(new MultiClusterData());
Assert.True(answer.IsEmpty);
var ts1 = new DateTime(year: 2011, month: 1, day: 1);
var ts2 = new DateTime(year: 2012, month: 2, day: 2);
var ts3 = new DateTime(year: 2013, month: 3, day: 3);
var conf1 = new MultiClusterConfiguration(ts1, new string[] { "A" }.ToList(), "comment");
var conf2 = new MultiClusterConfiguration(ts2, new string[] { "A", "B", "C" }.ToList());
var conf3 = new MultiClusterConfiguration(ts3, new string[] { }.ToList());
// push configuration 1
await gossipTable.Publish(new MultiClusterData(conf1));
// retrieve (by push/pull empty)
answer = await gossipTable.Synchronize(new MultiClusterData());
Assert.Equal(conf1, answer.Configuration);
// gossip stable
answer = await gossipTable.Synchronize(new MultiClusterData(conf1));
Assert.True(answer.IsEmpty);
// push configuration 2
answer = await gossipTable.Synchronize(new MultiClusterData(conf2));
Assert.True(answer.IsEmpty);
// gossip returns latest
answer = await gossipTable.Synchronize(new MultiClusterData(conf1));
Assert.Equal(conf2, answer.Configuration);
await gossipTable.Publish(new MultiClusterData(conf1));
answer = await gossipTable.Synchronize(new MultiClusterData());
Assert.Equal(conf2, answer.Configuration);
answer = await gossipTable.Synchronize(new MultiClusterData(conf2));
Assert.True(answer.IsEmpty);
// push final configuration
answer = await gossipTable.Synchronize(new MultiClusterData(conf3));
Assert.True(answer.IsEmpty);
answer = await gossipTable.Synchronize(new MultiClusterData(conf1));
Assert.Equal(conf3, answer.Configuration);
}
public void MultiClusterData_Configuration()
{
var ts1 = new DateTime(year: 2011, month: 1, day: 1);
var ts2 = new DateTime(year: 2012, month: 2, day: 2);
var conf1 = new MultiClusterConfiguration(ts1, new string[] { "A" }.ToList());
var conf2 = new MultiClusterConfiguration(ts2, new string[] { "A", "B", "C" }.ToList());
var gd1 = new MultiClusterData();
var gd2 = new MultiClusterData(conf1);
var gd3 = new MultiClusterData(conf2);
TestAlgebraicProperties(gd1, gd1);
TestAlgebraicProperties(gd2, gd2);
TestAlgebraicProperties(gd1, gd2);
TestAlgebraicProperties(gd3, gd1);
TestAlgebraicProperties(gd2, gd3);
TestAlgebraicProperties(gd3, gd2);
}
internal async Task<bool> TryUpdateConfigurationEntryAsync(MultiClusterConfiguration configuration, GossipTableEntry entry, string eTag)
{
if (configuration == null) throw new ArgumentNullException("configuration");
entry.GossipTimestamp = configuration.AdminTimestamp;
entry.Clusters = string.Join(GossipTableEntry.ClustersListSeparator, configuration.Clusters);
entry.Comment = configuration.Comment ?? "";
return (await TryUpdateTableEntryAsync(entry, eTag).ConfigureAwait(false));
}
internal async Task<bool> TryCreateConfigurationEntryAsync(MultiClusterConfiguration configuration)
{
if (configuration == null) throw new ArgumentNullException("configuration");
var entry = new GossipTableEntry
{
PartitionKey = GlobalServiceId,
RowKey = GossipTableEntry.CONFIGURATION_ROW,
GossipTimestamp = configuration.AdminTimestamp,
Clusters = string.Join(GossipTableEntry.ClustersListSeparator, configuration.Clusters),
Comment = configuration.Comment ?? ""
};
return (await TryCreateTableEntryAsync(entry).ConfigureAwait(false));
}
public async Task<MultiClusterConfiguration> InjectMultiClusterConfiguration(IEnumerable<string> clusters, string comment = "", bool checkForLaggingSilosFirst = true)
{
var multiClusterOracle = GetMultiClusterOracle();
var configuration = new MultiClusterConfiguration(DateTime.UtcNow, clusters.ToList(), comment);
if (!MultiClusterConfiguration.OlderThan(multiClusterOracle.GetMultiClusterConfiguration(), configuration))
throw new OrleansException("Could not inject multi-cluster configuration: current configuration is newer than clock");
if (checkForLaggingSilosFirst)
{
try
{
var laggingSilos = await multiClusterOracle.FindLaggingSilos(multiClusterOracle.GetMultiClusterConfiguration());
if (laggingSilos.Count > 0)
{
var msg = string.Format("Found unstable silos {0}", string.Join(",", laggingSilos));
throw new OrleansException(msg);
}
}
catch (Exception e)
{
throw new OrleansException("Could not inject multi-cluster configuration: stability check failed", e);
}
}
await multiClusterOracle.InjectMultiClusterConfiguration(configuration);
return configuration;
}
/// <summary>
/// Construct MultiClusterData containing a collection of gateway entries and a multi-cluster configuration.
/// </summary>
/// <param name="d">The gateway entries, by SiloAddress</param>
/// <param name="config">The configuration</param>
public MultiClusterData(IReadOnlyDictionary <SiloAddress, GatewayEntry> d, MultiClusterConfiguration config)
{
Gateways = d;
Configuration = config;
}
/// <summary>
/// Construct MultiClusterData containing a multi-cluster configuration.
/// </summary>
/// <param name="config">The configuration</param>
public MultiClusterData(MultiClusterConfiguration config)
{
Gateways = emptyd;
Configuration = config;
}
public async Task AzureGossip_ConfigGossip()
{
// start clean
await gossipTable.DeleteAllEntries();
// push empty data
await gossipTable.Publish(new MultiClusterData());
// push and pull empty data
var answer = await gossipTable.Synchronize(new MultiClusterData());
Assert.True(answer.IsEmpty);
var ts1 = new DateTime(year: 2011, month: 1, day: 1);
var ts2 = new DateTime(year: 2012, month: 2, day: 2);
var ts3 = new DateTime(year: 2013, month: 3, day: 3);
var conf1 = new MultiClusterConfiguration(ts1, new string[] { "A" }.ToList(), "comment");
var conf2 = new MultiClusterConfiguration(ts2, new string[] { "A", "B", "C" }.ToList());
var conf3 = new MultiClusterConfiguration(ts3, new string[] { }.ToList());
// push configuration 1
await gossipTable.Publish(new MultiClusterData(conf1));
// retrieve (by push/pull empty)
answer = await gossipTable.Synchronize(new MultiClusterData());
Assert.Equal(conf1, answer.Configuration);
// gossip stable
answer = await gossipTable.Synchronize(new MultiClusterData(conf1));
Assert.True(answer.IsEmpty);
// push configuration 2
answer = await gossipTable.Synchronize(new MultiClusterData(conf2));
Assert.True(answer.IsEmpty);
// gossip returns latest
answer = await gossipTable.Synchronize(new MultiClusterData(conf1));
Assert.Equal(conf2, answer.Configuration);
await gossipTable.Publish(new MultiClusterData(conf1));
answer = await gossipTable.Synchronize(new MultiClusterData());
Assert.Equal(conf2, answer.Configuration);
answer = await gossipTable.Synchronize(new MultiClusterData(conf2));
Assert.True(answer.IsEmpty);
// push final configuration
answer = await gossipTable.Synchronize(new MultiClusterData(conf3));
Assert.True(answer.IsEmpty);
answer = await gossipTable.Synchronize(new MultiClusterData(conf1));
Assert.Equal(conf3, answer.Configuration);
}
public static IEnumerable <string> GetRemoteInstances(this IMultiClusterRegistrationStrategy strategy, MultiClusterConfiguration configuration, string myClusterId)
{
return(strategy.GetRemoteInstances(configuration.Clusters, myClusterId));
}
// compare config with configInStorage, and
// - write config to storage if it is newer (or do nothing on etag conflict)
// - return config from store if it is newer
internal async Task<MultiClusterConfiguration> DiffAndWriteBackConfigAsync(MultiClusterConfiguration config, GossipTableEntry configInStorage)
{
// interpret empty admin timestamp by taking the azure table timestamp instead
// this allows an admin to inject a configuration by editing table directly
if (configInStorage != null && configInStorage.GossipTimestamp == default(DateTime))
configInStorage.GossipTimestamp = configInStorage.Timestamp.UtcDateTime;
if (config != null &&
(configInStorage == null || configInStorage.GossipTimestamp < config.AdminTimestamp))
{
// push the more recent configuration to storage
if (configInStorage == null)
await tableManager.TryCreateConfigurationEntryAsync(config);
else
await tableManager.TryUpdateConfigurationEntryAsync(config, configInStorage, configInStorage.ETag);
}
else if (configInStorage != null &&
(config == null || config.AdminTimestamp < configInStorage.GossipTimestamp))
{
// pull the more recent configuration from storage
return configInStorage.ToConfiguration();
}
return null;
}
/// <summary>
/// Called when configuration of the multicluster is changing.
/// </summary>
protected virtual Task OnConfigurationChange(MultiClusterConfiguration next)
{
Configuration = next;
return(TaskDone.Done);
}
public override IEnumerable <string> GetRemoteInstances(MultiClusterConfiguration mcConfig, string myClusterId)
{
return(emptyList); // there is only one instance, so no remote instances
}
/// <summary>
/// Called when configuration of the multicluster is changing.
/// </summary>
protected virtual Task OnConfigurationChange(MultiClusterConfiguration next)
{
Configuration = next;
return(Task.CompletedTask);
}
public void OnMultiClusterConfigurationChange(MultiClusterConfiguration next)
{
logger.Debug($"GSIP:M MultiClusterConfiguration {next}");
Prod();
}
public abstract IEnumerable <string> GetRemoteInstances(MultiClusterConfiguration mcConfig, string myClusterId);