public async Task Publish(IMultiClusterGossipData data)
{
logger.Debug("-Publish data:{0}", data);
// this is (almost) always called with just one item in data to be written back
// so we are o.k. with doing individual tasks for each storage read and write
var tasks = new List <Task>();
if (data.Configuration != null)
{
Func <Task> publishconfig = async() => {
var configInStorage = await tableManager.ReadConfigurationEntryAsync();
await DiffAndWriteBackConfigAsync(data.Configuration, configInStorage);
};
tasks.Add(publishconfig());
}
foreach (var gateway in data.Gateways.Values)
{
Func <Task> publishgatewayinfo = async() => {
var gatewayInfoInStorage = await tableManager.ReadGatewayEntryAsync(gateway);
await DiffAndWriteBackGatewayInfoAsync(gateway, gatewayInfoInStorage);
};
tasks.Add(publishgatewayinfo());
}
await Task.WhenAll(tasks);
}
private void PublishMyStatusToNewDestinations(IMultiClusterGossipData delta)
{
// for quicker convergence, we publish active local status information
// immediately when we learn about a new destination
GatewayEntry myEntry;
// don't do this if we are not an active gateway
if (!localData.Current.Gateways.TryGetValue(this.Silo, out myEntry) ||
myEntry.Status != GatewayStatus.Active)
{
return;
}
foreach (var gateway in delta.Gateways.Values)
{
var gossipworker = (gateway.ClusterId == this.clusterId) ?
GetSiloWorker(gateway.SiloAddress) : GetClusterWorker(gateway.ClusterId);
var destinationCluster = gateway.ClusterId;
if (!gossipworker.KnowsMe)
{
gossipworker.Publish(new MultiClusterData(myEntry));
}
}
}
public async Task <IMultiClusterGossipData> Synchronize(IMultiClusterGossipData pushed)
{
logger.Debug("-Synchronize pushed:{0}", pushed);
try
{
// read the entire table from storage
var entriesFromStorage = await tableManager.ReadAllEntriesAsync();
var configInStorage = entriesFromStorage.Item1;
var gatewayInfoInStorage = entriesFromStorage.Item2;
// diff and write back configuration
var configDeltaTask = DiffAndWriteBackConfigAsync(pushed.Configuration, configInStorage);
// diff and write back gateway info for each gateway appearing locally or in storage
var gatewayDeltaTasks = new List <Task <GatewayEntry> >();
var allAddresses = gatewayInfoInStorage.Keys.Union(pushed.Gateways.Keys);
foreach (var address in allAddresses)
{
GatewayEntry pushedInfo = null;
pushed.Gateways.TryGetValue(address, out pushedInfo);
GossipTableEntry infoInStorage = null;
gatewayInfoInStorage.TryGetValue(address, out infoInStorage);
gatewayDeltaTasks.Add(DiffAndWriteBackGatewayInfoAsync(pushedInfo, infoInStorage));
}
// wait for all the writeback tasks to complete
// these are not batched because we want them to fail individually on e-tag conflicts, not all
await configDeltaTask;
await Task.WhenAll(gatewayDeltaTasks);
// assemble delta pieces
var gw = new Dictionary <SiloAddress, GatewayEntry>();
foreach (var t in gatewayDeltaTasks)
{
var d = t.Result;
if (d != null)
{
gw.Add(d.SiloAddress, d);
}
}
var delta = new MultiClusterData(gw, configDeltaTask.Result);
logger.Debug("-Synchronize pulled delta:{0}", delta);
return(delta);
}
catch (Exception e)
{
logger.Info("-Synchronize encountered exception {0}", e);
throw e;
}
}
public IMultiClusterGossipData ApplyDataAndNotify(IMultiClusterGossipData data)
{
if (data.IsEmpty)
{
return(data);
}
MultiClusterData delta;
MultiClusterData prev = this.localData;
this.localData = prev.Merge(data, out delta);
if (logger.IsEnabled(LogLevel.Trace))
{
logger.Trace("ApplyDataAndNotify: delta {0}", delta);
}
if (delta.IsEmpty)
{
return(delta);
}
if (delta.Gateways.Count > 0)
{
// some gateways have changed
ComputeAvailableGatewaysPerCluster();
}
if (delta.Configuration != null)
{
// notify configuration listeners of change
List <IMultiClusterConfigurationListener> listenersToNotify;
lock (confListeners)
{
// make a copy under the lock
listenersToNotify = confListeners.ToList();
}
foreach (var listener in listenersToNotify)
{
try
{
listener.OnMultiClusterConfigurationChange(delta.Configuration);
}
catch (Exception exc)
{
logger.Error(ErrorCode.MultiClusterNetwork_LocalSubscriberException,
String.Format("IMultiClusterConfigurationListener {0} threw exception processing configuration {1}",
listener, delta.Configuration), exc);
}
}
}
return(delta);
}
public void Publish(IMultiClusterGossipData data)
{
// add the data to the data waiting to be published
toPublish = toPublish.Merge(data);
if (oracle.logger.IsEnabled(LogLevel.Debug))
{
LogQueuedPublish(toPublish);
}
Notify();
}
/// <summary>
/// incorporate source, producing new result, and report delta.
/// Ignores expired entries in source, and removes expired entries from this.
/// </summary>
/// <param name="source">The source data to apply to the data in this object</param>
/// <param name="delta">A delta of what changes were actually applied, used for change listeners</param>
/// <returns>The updated data</returns>
public MultiClusterData Merge(IMultiClusterGossipData source, out MultiClusterData delta)
{
//-- configuration
var sourceConf = source.Configuration;
var thisConf = this.Configuration;
MultiClusterConfiguration resultConf;
MultiClusterConfiguration deltaConf = null;
if (MultiClusterConfiguration.OlderThan(thisConf, sourceConf))
{
resultConf = sourceConf;
deltaConf = sourceConf;
}
else
{
resultConf = thisConf;
}
//-- gateways
var sourceList = source.Gateways;
var thisList = this.Gateways;
var resultList = new Dictionary <SiloAddress, GatewayEntry>();
var deltaList = new Dictionary <SiloAddress, GatewayEntry>();
foreach (var key in sourceList.Keys.Union(thisList.Keys).Distinct())
{
GatewayEntry thisEntry;
GatewayEntry sourceEntry;
thisList.TryGetValue(key, out thisEntry);
sourceList.TryGetValue(key, out sourceEntry);
if (sourceEntry != null && !sourceEntry.Expired &&
(thisEntry == null || thisEntry.HeartbeatTimestamp < sourceEntry.HeartbeatTimestamp))
{
resultList.Add(key, sourceEntry);
deltaList.Add(key, sourceEntry);
}
else if (thisEntry != null)
{
if (!thisEntry.Expired)
{
resultList.Add(key, thisEntry);
}
else
{
deltaList.Add(key, thisEntry);
}
}
}
delta = new MultiClusterData(deltaList, deltaConf);
return(new MultiClusterData(resultList, resultConf));
}
// called by remote nodes' full background gossip
public Task <IMultiClusterGossipData> Synchronize(IMultiClusterGossipData gossipData)
{
logger.Debug("--- Synchronize: gossip {0}", gossipData);
var data = (MultiClusterData)gossipData;
var delta = this.localData.ApplyDataAndNotify(data);
PublishMyStatusToNewDestinations(delta);
logger.Debug("--- Synchronize: done, answer={0}", delta);
return(Task.FromResult((IMultiClusterGossipData)delta));
}
// called by remote nodes that publish changes
public Task Publish(IMultiClusterGossipData gossipData, bool forwardLocally)
{
logger.Verbose("--- Publish: receive {0} data {1}", forwardLocally ? "remote" : "local", gossipData);
var data = (MultiClusterData)gossipData;
var delta = localData.ApplyDataAndNotify(data);
// forward changes to all local silos
if (forwardLocally)
{
foreach (var activeSilo in this.GetApproximateOtherActiveSilos())
GetSiloWorker(activeSilo).Publish(delta);
}
PublishMyStatusToNewDestinations(delta);
logger.Verbose("--- Publish: done");
return Task.CompletedTask;
}
/// <summary>
/// merge source into this object, and return result.
/// Ignores expired entries in source, and removes expired entries from this.
/// </summary>
/// <param name="source">The source data to apply to the data in this object</param>
/// <returns>The updated data</returns>
public MultiClusterData Merge(IMultiClusterGossipData source)
{
MultiClusterData ignore;
return(Merge(source, out ignore));
}
public IMultiClusterGossipData ApplyDataAndNotify(IMultiClusterGossipData data)
{
if (data.IsEmpty)
{
return(data);
}
MultiClusterData delta;
MultiClusterData prev = this.localData;
this.localData = prev.Merge(data, out delta);
if (logger.IsEnabled(LogLevel.Trace))
{
logger.Trace("ApplyDataAndNotify: delta {0}", delta);
}
if (delta.IsEmpty)
{
return(delta);
}
if (delta.Gateways.Count > 0)
{
// some gateways have changed
ComputeAvailableGatewaysPerCluster();
}
if (delta.Configuration != null)
{
// notify configuration listeners of change
List <GrainReference> listenersToNotify;
lock (confListeners)
{
// make a copy under the lock
listenersToNotify = confListeners.ToList();
}
foreach (var listener in listenersToNotify)
{
try
{
if (logger.IsEnabled(LogLevel.Trace))
{
logger.Trace("-NotificationWork: notify IProtocolParticipant {0} of configuration {1}", listener, delta.Configuration);
}
// enqueue conf change event as grain call
var g = this.grainFactory.Cast <ILogConsistencyProtocolParticipant>(listener);
g.OnMultiClusterConfigurationChange(delta.Configuration).Ignore();
}
catch (Exception exc)
{
logger.Error(ErrorCode.MultiClusterNetwork_LocalSubscriberException,
String.Format("IProtocolParticipant {0} threw exception processing configuration {1}",
listener, delta.Configuration), exc);
}
}
}
return(delta);
}